From b982d987a7b95fd60442dceb8fd71f8b1dc869ee Mon Sep 17 00:00:00 2001 From: Megan Marsh Date: Fri, 15 May 2020 13:13:43 -0700 Subject: [PATCH 1/3] Add Artifice postprocessor ID to valid artifacts for postprocessors that perform artifact type validation --- post-processor/googlecompute-export/post-processor.go | 8 ++++++-- post-processor/googlecompute-import/post-processor.go | 10 +++++++--- post-processor/vagrant-cloud/post-processor.go | 2 +- post-processor/vagrant/post-processor.go | 2 ++ post-processor/vsphere-template/post-processor.go | 2 ++ post-processor/yandex-export/post-processor.go | 8 ++++++-- 6 files changed, 24 insertions(+), 8 deletions(-) diff --git a/post-processor/googlecompute-export/post-processor.go b/post-processor/googlecompute-export/post-processor.go index 2d4a2a995..7341e6cfb 100644 --- a/post-processor/googlecompute-export/post-processor.go +++ b/post-processor/googlecompute-export/post-processor.go @@ -14,6 +14,7 @@ import ( "github.com/hashicorp/packer/helper/config" "github.com/hashicorp/packer/helper/multistep" "github.com/hashicorp/packer/packer" + "github.com/hashicorp/packer/post-processor/artifice" "github.com/hashicorp/packer/template/interpolate" "golang.org/x/oauth2/jwt" ) @@ -92,9 +93,12 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { } func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, bool, error) { - if artifact.BuilderId() != googlecompute.BuilderId { + switch artifact.BuilderId() { + case googlecompute.BuilderId, artifice.BuilderId: + break + default: err := fmt.Errorf( - "Unknown artifact type: %s\nCan only export from Google Compute Engine builder artifacts.", + "Unknown artifact type: %s\nCan only export from Google Compute Engine builder and Artifice post-processor artifacts.", artifact.BuilderId()) return nil, false, false, err } diff --git a/post-processor/googlecompute-import/post-processor.go b/post-processor/googlecompute-import/post-processor.go index 33f2501b9..2379e3e3c 100644 --- a/post-processor/googlecompute-import/post-processor.go +++ b/post-processor/googlecompute-import/post-processor.go @@ -19,6 +19,7 @@ import ( "github.com/hashicorp/packer/common" "github.com/hashicorp/packer/helper/config" "github.com/hashicorp/packer/packer" + "github.com/hashicorp/packer/post-processor/artifice" "github.com/hashicorp/packer/post-processor/compress" "github.com/hashicorp/packer/template/interpolate" ) @@ -123,9 +124,12 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact return nil, false, false, err } - if artifact.BuilderId() != compress.BuilderId { - err = fmt.Errorf( - "incompatible artifact type: %s\nCan only import from Compress post-processor artifacts", + switch artifact.BuilderId() { + case compress.BuilderId, artifice.BuilderId: + break + default: + err := fmt.Errorf( + "Unknown artifact type: %s\nCan only import from Compress post-processor and Artifice post-processor artifacts.", artifact.BuilderId()) return nil, false, false, err } diff --git a/post-processor/vagrant-cloud/post-processor.go b/post-processor/vagrant-cloud/post-processor.go index 26a30cc1a..b79652a67 100644 --- a/post-processor/vagrant-cloud/post-processor.go +++ b/post-processor/vagrant-cloud/post-processor.go @@ -130,7 +130,7 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, bool, error) { if _, ok := builtins[artifact.BuilderId()]; !ok { return nil, false, false, fmt.Errorf( - "Unknown artifact type, requires box from vagrant post-processor or vagrant builder: %s", artifact.BuilderId()) + "Unknown artifact type: this post-processor requires an input artifact from the artifice post-processor, vagrant post-processor, or vagrant builder: %s", artifact.BuilderId()) } // We assume that there is only one .box file to upload diff --git a/post-processor/vagrant/post-processor.go b/post-processor/vagrant/post-processor.go index 15faa0c9e..624bb7b0e 100644 --- a/post-processor/vagrant/post-processor.go +++ b/post-processor/vagrant/post-processor.go @@ -19,6 +19,7 @@ import ( "github.com/hashicorp/packer/helper/config" "github.com/hashicorp/packer/packer" "github.com/hashicorp/packer/packer/tmp" + "github.com/hashicorp/packer/post-processor/artifice" "github.com/hashicorp/packer/template/interpolate" "github.com/mitchellh/mapstructure" ) @@ -40,6 +41,7 @@ var builtins = map[string]string{ "packer.post-processor.docker-import": "docker", "packer.post-processor.docker-tag": "docker", "packer.post-processor.docker-push": "docker", + artifice.BuilderId: "artifice", } type Config struct { diff --git a/post-processor/vsphere-template/post-processor.go b/post-processor/vsphere-template/post-processor.go index 490453e1b..978970633 100644 --- a/post-processor/vsphere-template/post-processor.go +++ b/post-processor/vsphere-template/post-processor.go @@ -19,6 +19,7 @@ import ( "github.com/hashicorp/packer/helper/config" "github.com/hashicorp/packer/helper/multistep" "github.com/hashicorp/packer/packer" + "github.com/hashicorp/packer/post-processor/artifice" "github.com/hashicorp/packer/template/interpolate" "github.com/vmware/govmomi" ) @@ -27,6 +28,7 @@ var builtins = map[string]string{ vspherepost.BuilderId: "vmware", vmwcommon.BuilderIdESX: "vmware", vsphere.BuilderId: "vsphere", + artifice.BuilderId: "artifice", } type Config struct { diff --git a/post-processor/yandex-export/post-processor.go b/post-processor/yandex-export/post-processor.go index 1a37bb16b..7987e9d78 100644 --- a/post-processor/yandex-export/post-processor.go +++ b/post-processor/yandex-export/post-processor.go @@ -16,6 +16,7 @@ import ( "github.com/hashicorp/packer/helper/config" "github.com/hashicorp/packer/helper/multistep" "github.com/hashicorp/packer/packer" + "github.com/hashicorp/packer/post-processor/artifice" "github.com/hashicorp/packer/template/interpolate" ) @@ -106,9 +107,12 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { } func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, bool, error) { - if artifact.BuilderId() != yandex.BuilderID { + switch artifact.BuilderId() { + case yandex.BuilderID, artifice.BuilderId: + break + default: err := fmt.Errorf( - "Unknown artifact type: %s\nCan only export from Yandex Cloud builder artifacts.", + "Unknown artifact type: %s\nCan only export from Yandex Cloud builder artifact or Artifice post-processor artifact.", artifact.BuilderId()) return nil, false, false, err } From 557eb015ea4ae4bafe3fd297d464db5decf066a6 Mon Sep 17 00:00:00 2001 From: Megan Marsh Date: Tue, 19 May 2020 15:52:12 -0700 Subject: [PATCH 2/3] make artifice post-processor play nicely with vagrant post-processor, including template validation., Update docs for each. --- post-processor/vagrant/post-processor.go | 58 ++++++++++++++++--- post-processor/vagrant/post-processor_test.go | 18 ++++++ .../pages/docs/post-processors/artifice.mdx | 18 +++--- .../pages/docs/post-processors/vagrant.mdx | 14 +++++ 4 files changed, 92 insertions(+), 16 deletions(-) diff --git a/post-processor/vagrant/post-processor.go b/post-processor/vagrant/post-processor.go index 624bb7b0e..a362b2e46 100644 --- a/post-processor/vagrant/post-processor.go +++ b/post-processor/vagrant/post-processor.go @@ -12,6 +12,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strings" "text/template" "github.com/hashicorp/hcl/v2/hcldec" @@ -41,7 +42,21 @@ var builtins = map[string]string{ "packer.post-processor.docker-import": "docker", "packer.post-processor.docker-tag": "docker", "packer.post-processor.docker-push": "docker", - artifice.BuilderId: "artifice", +} + +func availableProviders() []string { + dedupedProvidersMap := map[string]string{} + + for _, v := range builtins { + dedupedProvidersMap[v] = v + } + + dedupedProviders := []string{} + for k := range dedupedProvidersMap { + dedupedProviders = append(dedupedProviders, k) + } + + return dedupedProviders } type Config struct { @@ -53,6 +68,7 @@ type Config struct { Override map[string]interface{} VagrantfileTemplate string `mapstructure:"vagrantfile_template"` VagrantfileTemplateGenerated bool `mapstructure:"vagrantfile_template_generated"` + ProviderOverride string `mapstructure:"provider_override"` ctx interpolate.Context } @@ -69,6 +85,22 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { if err := p.configureSingle(&p.config, raws...); err != nil { return err } + + if p.config.ProviderOverride != "" { + validOverride := false + providers := availableProviders() + for _, prov := range providers { + if prov == p.config.ProviderOverride { + validOverride = true + break + } + } + if !validOverride { + return fmt.Errorf("The given provider_override %s is not valid. "+ + "Please choose from one of %s", p.config.ProviderOverride, + strings.Join(providers, ", ")) + } + } return nil } @@ -170,17 +202,27 @@ func (p *PostProcessor) PostProcessProvider(name string, provider Provider, ui p } func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, bool, error) { - - name, ok := builtins[artifact.BuilderId()] - if !ok { - return nil, false, false, fmt.Errorf( - "Unknown artifact type, can't build box: %s", artifact.BuilderId()) + name := p.config.ProviderOverride + if name == "" { + n, ok := builtins[artifact.BuilderId()] + if !ok { + return nil, false, false, fmt.Errorf( + "Unknown artifact type, can't build box: %s", artifact.BuilderId()) + } + name = n } provider := providerForName(name) if provider == nil { - // This shouldn't happen since we hard code all of these ourselves - panic(fmt.Sprintf("bad provider name: %s", name)) + if artifact.BuilderId() == artifice.BuilderId { + return nil, false, false, fmt.Errorf( + "Unknown provider type: When using an artifact created by " + + "the artifice post-processor, you need to set the " + + "provider_override option.") + } else { + // This shouldn't happen since we hard code all of these ourselves + panic(fmt.Sprintf("bad provider name: %s", name)) + } } artifact, keep, err := p.PostProcessProvider(name, provider, ui, artifact) diff --git a/post-processor/vagrant/post-processor_test.go b/post-processor/vagrant/post-processor_test.go index 8d414bab2..2a86821c4 100644 --- a/post-processor/vagrant/post-processor_test.go +++ b/post-processor/vagrant/post-processor_test.go @@ -164,6 +164,24 @@ func TestPostProcessorPrepare_vagrantfileTemplateExists(t *testing.T) { } } +func TestPostProcessorPrepare_ProviderOverrideExists(t *testing.T) { + c := testConfig() + c["provider_override"] = "foo" + + var p PostProcessor + + if err := p.Configure(c); err == nil { + t.Fatal("Should have errored since foo is not a valid vagrant provider") + } + + c = testConfig() + c["provider_override"] = "aws" + + if err := p.Configure(c); err != nil { + t.Fatal("Should not have errored since aws is a valid vagrant provider") + } +} + func TestPostProcessorPostProcess_badId(t *testing.T) { artifact := &packer.MockArtifact{ BuilderIdValue: "invalid.packer", diff --git a/website/pages/docs/post-processors/artifice.mdx b/website/pages/docs/post-processors/artifice.mdx index c57704420..8d0ecd49b 100644 --- a/website/pages/docs/post-processors/artifice.mdx +++ b/website/pages/docs/post-processors/artifice.mdx @@ -23,16 +23,18 @@ Type: `artifice` The artifice post-processor overrides the artifact list from an upstream builder or post-processor. All downstream post-processors will see the new -artifacts you specify. The primary use-case is to build artifacts inside a -packer builder -- for example, spinning up an EC2 instance to build a docker -container -- and then extracting the docker container and throwing away the EC2 -instance. +artifacts you specify. After overriding the artifact with artifice, you can use it with other -post-processors like -[compress](/docs/post-processors/compress), -[docker-push](/docs/post-processors/docker-push), or -a third-party post-processor. +post-processors, including most of the core post-processors and third-party +post-processors. + +A major benefit of this is that you can modify builder +artifacts using shell-local and pass those modified artifacts into +post-processors that may not have worked with the original builder. +For example, maybe you want to export a docker container from an amazon-ebs +builder and then use Docker-push to put that Docker container into your Docker +Hub account. Artifice allows you to use the familiar packer workflow to create a fresh, stateless build environment for each build on the infrastructure of your diff --git a/website/pages/docs/post-processors/vagrant.mdx b/website/pages/docs/post-processors/vagrant.mdx index 5de485177..b98b659f4 100644 --- a/website/pages/docs/post-processors/vagrant.mdx +++ b/website/pages/docs/post-processors/vagrant.mdx @@ -94,6 +94,13 @@ more details about certain options in following sections. By default, the value of this config is `packer_{{.BuildName}}_{{.Provider}}.box`. +- `provider_override` (string) - this option will override the internal logic + that decides which Vagrant provider to set for a particular Packer builder's + or post-processor's artifact. It is required when the artifact comes from the + Artifice post-processor, but is otherwise optional. Valid options are: + `digitalocean`, `virtualbox`, `azure`, `vmware`, `libvirt`, `docker`, + `lxc`, `scaleway`, `hyperv`, `parallels`, `aws`, or `google`. + - `vagrantfile_template` (string) - Path to a template to use for the Vagrantfile that is packaged with the box. @@ -179,3 +186,10 @@ accelerators: none, kvm, tcg, or hvf. If you are using the Vagrant post-processor with the `vmware-esxi` builder, you must export the builder artifact locally; the Vagrant post-processor will not work on remote artifacts. + +### Artifice + +If you are using this post-processor after defining an artifact using the +Artifice post-processor, then you must set the "provider_override" template +option so that the Vagrant post-processor knows what provider to use to create +the Vagrant box. From 4041cb77757cea034b73ad184df29f5a27282667 Mon Sep 17 00:00:00 2001 From: Megan Marsh Date: Tue, 19 May 2020 16:10:51 -0700 Subject: [PATCH 3/3] generated code --- post-processor/vagrant/post-processor.hcl2spec.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/post-processor/vagrant/post-processor.hcl2spec.go b/post-processor/vagrant/post-processor.hcl2spec.go index b72c2c85a..585fc2c5b 100644 --- a/post-processor/vagrant/post-processor.hcl2spec.go +++ b/post-processor/vagrant/post-processor.hcl2spec.go @@ -22,6 +22,7 @@ type FlatConfig struct { Override map[string]interface{} `cty:"override"` VagrantfileTemplate *string `mapstructure:"vagrantfile_template" cty:"vagrantfile_template"` VagrantfileTemplateGenerated *bool `mapstructure:"vagrantfile_template_generated" cty:"vagrantfile_template_generated"` + ProviderOverride *string `mapstructure:"provider_override" cty:"provider_override"` } // FlatMapstructure returns a new FlatConfig. @@ -49,6 +50,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec { "override": &hcldec.AttrSpec{Name: "override", Type: cty.Map(cty.String), Required: false}, "vagrantfile_template": &hcldec.AttrSpec{Name: "vagrantfile_template", Type: cty.String, Required: false}, "vagrantfile_template_generated": &hcldec.AttrSpec{Name: "vagrantfile_template_generated", Type: cty.Bool, Required: false}, + "provider_override": &hcldec.AttrSpec{Name: "provider_override", Type: cty.String, Required: false}, } return s }