Merge pull request #8018 from DanHam/vagrant-cloud-pp-allow-artifice

Allow use of the Artifice post-processor with the Vagrant Cloud post-processor

post-processor/vagrant-cloud/post-processor.go

@@ -5,8 +5,14 @@
 package vagrantcloud
 
 import (
+	"archive/tar"
+	"compress/gzip"
 	"context"
+	"encoding/json"
 	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
 	"os"
 	"strings"
 
@@ -19,6 +25,7 @@ import (
 
 var builtins = map[string]string{
 	"mitchellh.post-processor.vagrant": "vagrant",
+	"packer.post-processor.artifice":   "artifice",
 	"vagrant":                          "vagrant",
 }
 
@@ -89,7 +96,7 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
 	// Accumulate any errors
 	errs := new(packer.MultiError)
 
-	// required configuration
+	// Required configuration
 	templates := map[string]*string{
 		"box_tag":      &p.config.Tag,
 		"version":      &p.config.Version,
@@ -103,7 +110,7 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
 		}
 	}
 
-	// create the HTTP client
+	// Create the HTTP client
 	p.client, err = VagrantCloudClient{}.New(p.config.VagrantCloudUrl, p.config.AccessToken, p.insecureSkipTLSVerify)
 	if err != nil {
 		errs = packer.MultiErrorAppend(
@@ -126,20 +133,21 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
 	// We assume that there is only one .box file to upload
 	if !strings.HasSuffix(artifact.Files()[0], ".box") {
 		return nil, false, false, fmt.Errorf(
-			"Unknown files in artifact, vagrant box is required: %s", artifact.Files())
+			"Unknown files in artifact, Vagrant box with .box suffix is required as first artifact file: %s", artifact.Files())
 	}
 
 	if p.warnAtlasToken {
 		ui.Message("Warning: Using Vagrant Cloud token found in ATLAS_TOKEN. Please make sure it is correct, or set VAGRANT_CLOUD_TOKEN")
 	}
 
-	// The name of the provider for vagrant cloud, and vagrant
+	// Determine the name of the provider for Vagrant Cloud, and Vagrant
-	providerName := providerFromBuilderName(artifact.Id())
+	providerName, err := getProvider(artifact.Id(), artifact.Files()[0], builtins[artifact.BuilderId()])
 
 	p.config.ctx.Data = &boxDownloadUrlTemplate{
 		ArtifactId: artifact.Id(),
 		Provider:   providerName,
 	}
+
 	boxDownloadUrl, err := interpolate.Render(p.config.BoxDownloadUrl, &p.config.ctx)
 	if err != nil {
 		return nil, false, false, fmt.Errorf("Error processing box_download_url: %s", err)
@@ -187,8 +195,21 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
 	return NewArtifact(providerName, p.config.Tag), true, false, nil
 }
 
-// converts a packer builder name to the corresponding vagrant
+func getProvider(builderName, boxfile, builderId string) (providerName string, err error) {
-// provider
+	if builderId == "artifice" {
+		// The artifice post processor cannot embed any data in the
+		// supplied artifact so the provider information must be extracted
+		// from the box file directly
+		providerName, err = providerFromVagrantBox(boxfile)
+	} else {
+		// For the Vagrant builder and Vagrant post processor the provider can
+		// be determined from information embedded in the artifact
+		providerName = providerFromBuilderName(builderName)
+	}
+	return providerName, err
+}
+
+// Converts a packer builder name to the corresponding vagrant provider
 func providerFromBuilderName(name string) string {
 	switch name {
 	case "aws":
@@ -207,3 +228,59 @@ func providerFromBuilderName(name string) string {
 		return name
 	}
 }
+
+// Returns the Vagrant provider the box is intended for use with by
+// reading the metadata file packaged inside the box
+func providerFromVagrantBox(boxfile string) (providerName string, err error) {
+	log.Println("Attempting to determine provider from metadata in box file. This may take some time...")
+
+	f, err := os.Open(boxfile)
+	if err != nil {
+		return "", fmt.Errorf("Error attempting to open box file: %s", err)
+	}
+	defer f.Close()
+
+	// Vagrant boxes are gzipped tar archives
+	ar, err := gzip.NewReader(f)
+	if err != nil {
+		return "", fmt.Errorf("Error unzipping box archive: %s", err)
+	}
+	tr := tar.NewReader(ar)
+
+	// The metadata.json file in the tar archive contains a 'provider' key
+	type metadata struct {
+		ProviderName string `json:"provider"`
+	}
+	md := metadata{}
+
+	// Loop through the files in the archive and read the provider
+	// information from the boxes metadata.json file
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			if md.ProviderName == "" {
+				return "", fmt.Errorf("Error: Provider info was not found in box: %s", boxfile)
+			}
+			break
+		}
+		if err != nil {
+			return "", fmt.Errorf("Error reading header info from box tar archive: %s", err)
+		}
+
+		if hdr.Name == "metadata.json" {
+			contents, err := ioutil.ReadAll(tr)
+			if err != nil {
+				return "", fmt.Errorf("Error reading contents of metadata.json file from box file: %s", err)
+			}
+			err = json.Unmarshal(contents, &md)
+			if err != nil {
+				return "", fmt.Errorf("Error parsing metadata.json file: %s", err)
+			}
+			if md.ProviderName == "" {
+				return "", fmt.Errorf("Error: Could not determine Vagrant provider from box metadata.json file")
+			}
+			break
+		}
+	}
+	return md.ProviderName, nil
+}

post-processor/vagrant-cloud/post-processor_test.go

@@ -1,16 +1,26 @@
 package vagrantcloud
 
 import (
+	"archive/tar"
 	"bytes"
+	"compress/gzip"
+	"context"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/hashicorp/packer/packer"
+	"github.com/stretchr/testify/assert"
 )
 
+type tarFiles []struct {
+	Name, Body string
+}
+
 func testGoodConfig() map[string]interface{} {
 	return map[string]interface{}{
 		"access_token":        "foo",
@@ -137,6 +147,44 @@ func TestPostProcessor_Configure_Bad(t *testing.T) {
 	}
 }
 
+func TestPostProcessor_PostProcess_checkArtifactType(t *testing.T) {
+	artifact := &packer.MockArtifact{
+		BuilderIdValue: "invalid.builder",
+	}
+
+	config := testGoodConfig()
+	server := newSecureServer("foo", nil)
+	defer server.Close()
+	config["vagrant_cloud_url"] = server.URL
+	var p PostProcessor
+
+	p.Configure(config)
+	_, _, _, err := p.PostProcess(context.Background(), testUi(), artifact)
+	if !strings.Contains(err.Error(), "Unknown artifact type") {
+		t.Fatalf("Should error with message 'Unknown artifact type...' with BuilderId: %s", artifact.BuilderIdValue)
+	}
+}
+
+func TestPostProcessor_PostProcess_checkArtifactFileIsBox(t *testing.T) {
+	artifact := &packer.MockArtifact{
+		BuilderIdValue: "mitchellh.post-processor.vagrant", // good
+		FilesValue:     []string{"invalid.boxfile"},        // should have .box extension
+	}
+
+	config := testGoodConfig()
+	server := newSecureServer("foo", nil)
+	defer server.Close()
+	config["vagrant_cloud_url"] = server.URL
+	var p PostProcessor
+
+	p.Configure(config)
+	_, _, _, err := p.PostProcess(context.Background(), testUi(), artifact)
+	if !strings.Contains(err.Error(), "Unknown files in artifact") {
+		t.Fatalf("Should error with message 'Unknown files in artifact...' with artifact file: %s",
+			artifact.FilesValue[0])
+	}
+}
+
 func testUi() *packer.BasicUi {
 	return &packer.BasicUi{
 		Reader: new(bytes.Buffer),
@@ -157,3 +205,258 @@ func TestProviderFromBuilderName(t *testing.T) {
 		t.Fatal("should convert provider")
 	}
 }
+
+func TestProviderFromVagrantBox_missing_box(t *testing.T) {
+	// Bad: Box does not exist
+	boxfile := "i_dont_exist.box"
+	_, err := providerFromVagrantBox(boxfile)
+	if err == nil {
+		t.Fatal("Should have error as box file does not exist")
+	}
+	t.Logf("%s", err)
+}
+
+func TestProviderFromVagrantBox_empty_box(t *testing.T) {
+	// Bad: Empty box file
+	boxfile, err := newBoxFile()
+	if err != nil {
+		t.Fatalf("%s", err)
+	}
+	defer os.Remove(boxfile.Name())
+
+	_, err = providerFromVagrantBox(boxfile.Name())
+	if err == nil {
+		t.Fatal("Should have error as box file is empty")
+	}
+	t.Logf("%s", err)
+}
+
+func TestProviderFromVagrantBox_gzip_only_box(t *testing.T) {
+	boxfile, err := newBoxFile()
+	if err != nil {
+		t.Fatalf("%s", err)
+	}
+	defer os.Remove(boxfile.Name())
+
+	// Bad: Box is just a plain gzip file
+	aw := gzip.NewWriter(boxfile)
+	_, err = aw.Write([]byte("foo content"))
+	if err != nil {
+		t.Fatal("Error zipping test box file")
+	}
+	aw.Close() // Flush the gzipped contents to file
+
+	_, err = providerFromVagrantBox(boxfile.Name())
+	if err == nil {
+		t.Fatalf("Should have error as box file is a plain gzip file: %s", err)
+	}
+	t.Logf("%s", err)
+}
+
+func TestProviderFromVagrantBox_no_files_in_archive(t *testing.T) {
+	// Bad: Box contains no files
+	boxfile, err := createBox(tarFiles{})
+	if err != nil {
+		t.Fatalf("Error creating test box: %s", err)
+	}
+	defer os.Remove(boxfile.Name())
+
+	_, err = providerFromVagrantBox(boxfile.Name())
+	if err == nil {
+		t.Fatalf("Should have error as box file has no contents")
+	}
+	t.Logf("%s", err)
+}
+
+func TestProviderFromVagrantBox_no_metadata(t *testing.T) {
+	// Bad: Box contains no metadata/metadata.json file
+	files := tarFiles{
+		{"foo.txt", "This is a foo file"},
+		{"bar.txt", "This is a bar file"},
+	}
+	boxfile, err := createBox(files)
+	if err != nil {
+		t.Fatalf("Error creating test box: %s", err)
+	}
+	defer os.Remove(boxfile.Name())
+
+	_, err = providerFromVagrantBox(boxfile.Name())
+	if err == nil {
+		t.Fatalf("Should have error as box file does not include metadata.json file")
+	}
+	t.Logf("%s", err)
+}
+
+func TestProviderFromVagrantBox_metadata_empty(t *testing.T) {
+	// Bad: Create a box with an empty metadata.json file
+	files := tarFiles{
+		{"foo.txt", "This is a foo file"},
+		{"bar.txt", "This is a bar file"},
+		{"metadata.json", ""},
+	}
+	boxfile, err := createBox(files)
+	if err != nil {
+		t.Fatalf("Error creating test box: %s", err)
+	}
+	defer os.Remove(boxfile.Name())
+
+	_, err = providerFromVagrantBox(boxfile.Name())
+	if err == nil {
+		t.Fatalf("Should have error as box files metadata.json file is empty")
+	}
+	t.Logf("%s", err)
+}
+
+func TestProviderFromVagrantBox_metadata_bad_json(t *testing.T) {
+	// Bad: Create a box with bad JSON in the metadata.json file
+	files := tarFiles{
+		{"foo.txt", "This is a foo file"},
+		{"bar.txt", "This is a bar file"},
+		{"metadata.json", "{provider: badjson}"},
+	}
+	boxfile, err := createBox(files)
+	if err != nil {
+		t.Fatalf("Error creating test box: %s", err)
+	}
+	defer os.Remove(boxfile.Name())
+
+	_, err = providerFromVagrantBox(boxfile.Name())
+	if err == nil {
+		t.Fatalf("Should have error as box files metadata.json file is empty")
+	}
+	t.Logf("%s", err)
+}
+
+func TestProviderFromVagrantBox_metadata_no_provider_key(t *testing.T) {
+	// Bad: Create a box with no 'provider' key in the metadata.json file
+	files := tarFiles{
+		{"foo.txt", "This is a foo file"},
+		{"bar.txt", "This is a bar file"},
+		{"metadata.json", `{"cows":"moo"}`},
+	}
+	boxfile, err := createBox(files)
+	if err != nil {
+		t.Fatalf("Error creating test box: %s", err)
+	}
+	defer os.Remove(boxfile.Name())
+
+	_, err = providerFromVagrantBox(boxfile.Name())
+	if err == nil {
+		t.Fatalf("Should have error as box files metadata.json file is empty")
+	}
+	t.Logf("%s", err)
+}
+
+func TestProviderFromVagrantBox_metadata_provider_value_empty(t *testing.T) {
+	// Bad: The boxes metadata.json file 'provider' key has an empty value
+	files := tarFiles{
+		{"foo.txt", "This is a foo file"},
+		{"bar.txt", "This is a bar file"},
+		{"metadata.json", `{"provider":""}`},
+	}
+	boxfile, err := createBox(files)
+	if err != nil {
+		t.Fatalf("Error creating test box: %s", err)
+	}
+	defer os.Remove(boxfile.Name())
+
+	_, err = providerFromVagrantBox(boxfile.Name())
+	if err == nil {
+		t.Fatalf("Should have error as boxes metadata.json file 'provider' key is empty")
+	}
+	t.Logf("%s", err)
+}
+
+func TestProviderFromVagrantBox_metadata_ok(t *testing.T) {
+	// Good: The boxes metadata.json file has the 'provider' key/value pair
+	expectedProvider := "virtualbox"
+	files := tarFiles{
+		{"foo.txt", "This is a foo file"},
+		{"bar.txt", "This is a bar file"},
+		{"metadata.json", `{"provider":"` + expectedProvider + `"}`},
+	}
+	boxfile, err := createBox(files)
+	if err != nil {
+		t.Fatalf("Error creating test box: %s", err)
+	}
+	defer os.Remove(boxfile.Name())
+
+	provider, err := providerFromVagrantBox(boxfile.Name())
+	assert.Equal(t, expectedProvider, provider, "Error: Expected provider: '%s'. Got '%s'", expectedProvider, provider)
+	t.Logf("Expected provider '%s'. Got provider '%s'", expectedProvider, provider)
+}
+
+func TestGetProvider_artifice(t *testing.T) {
+	expectedProvider := "virtualbox"
+	files := tarFiles{
+		{"foo.txt", "This is a foo file"},
+		{"bar.txt", "This is a bar file"},
+		{"metadata.json", `{"provider":"` + expectedProvider + `"}`},
+	}
+	boxfile, err := createBox(files)
+	if err != nil {
+		t.Fatalf("Error creating test box: %s", err)
+	}
+	defer os.Remove(boxfile.Name())
+
+	provider, err := getProvider("", boxfile.Name(), "artifice")
+	assert.Equal(t, expectedProvider, provider, "Error: Expected provider: '%s'. Got '%s'", expectedProvider, provider)
+	t.Logf("Expected provider '%s'. Got provider '%s'", expectedProvider, provider)
+}
+
+func TestGetProvider_other(t *testing.T) {
+	expectedProvider := "virtualbox"
+
+	provider, _ := getProvider(expectedProvider, "foo.box", "other")
+	assert.Equal(t, expectedProvider, provider, "Error: Expected provider: '%s'. Got '%s'", expectedProvider, provider)
+	t.Logf("Expected provider '%s'. Got provider '%s'", expectedProvider, provider)
+}
+
+func newBoxFile() (boxfile *os.File, err error) {
+	boxfile, err = ioutil.TempFile(os.TempDir(), "test*.box")
+	if err != nil {
+		return boxfile, fmt.Errorf("Error creating test box file: %s", err)
+	}
+	return boxfile, nil
+}
+
+func createBox(files tarFiles) (boxfile *os.File, err error) {
+	boxfile, err = newBoxFile()
+	if err != nil {
+		return boxfile, err
+	}
+
+	// Box files are gzipped tar archives
+	aw := gzip.NewWriter(boxfile)
+	tw := tar.NewWriter(aw)
+
+	// Add each file to the box
+	for _, file := range files {
+		// Create and write the tar file header
+		hdr := &tar.Header{
+			Name: file.Name,
+			Mode: 0644,
+			Size: int64(len(file.Body)),
+		}
+		err = tw.WriteHeader(hdr)
+		if err != nil {
+			return boxfile, fmt.Errorf("Error writing box tar file header: %s", err)
+		}
+		// Write the file contents
+		_, err = tw.Write([]byte(file.Body))
+		if err != nil {
+			return boxfile, fmt.Errorf("Error writing box tar file contents: %s", err)
+		}
+	}
+	// Flush and close each writer
+	err = tw.Close()
+	if err != nil {
+		return boxfile, fmt.Errorf("Error flushing tar file contents: %s", err)
+	}
+	err = aw.Close()
+	if err != nil {
+		return boxfile, fmt.Errorf("Error flushing gzip file contents: %s", err)
+	}
+
+	return boxfile, nil
+}

website/source/docs/post-processors/vagrant-cloud.html.md

@@ -1,9 +1,7 @@
 ---
 description: |
-    The Packer Vagrant Cloud post-processor receives a Vagrant box from the
+    The Vagrant Cloud post-processor enables the upload of Vagrant boxes to
-    `vagrant` post-processor or vagrant builder and pushes it to Vagrant Cloud.
+    Vagrant Cloud.
-    Vagrant Cloud hosts and serves boxes to Vagrant, allowing you to version and
-    distribute boxes to an organization in a simple way.
 layout: docs
 page_title: 'Vagrant Cloud - Post-Processors'
 sidebar_current: 'docs-post-processors-vagrant-cloud'
@@ -13,12 +11,16 @@ sidebar_current: 'docs-post-processors-vagrant-cloud'
 
 Type: `vagrant-cloud`
 
-The Packer Vagrant Cloud post-processor receives a Vagrant box from the
-`vagrant` post-processor or vagrant builder and pushes it to Vagrant Cloud.
 [Vagrant Cloud](https://app.vagrantup.com/boxes/search) hosts and serves boxes
 to Vagrant, allowing you to version and distribute boxes to an organization in a
 simple way.
 
+The Vagrant Cloud post-processor enables the upload of Vagrant boxes to Vagrant
+Cloud. Currently, the Vagrant Cloud post-processor will accept and upload boxes
+supplied to it from the [Vagrant](/docs/post-processors/vagrant.html) or
+[Artifice](/docs/post-processors/artifice.html) post-processors and the
+[Vagrant](/docs/builders/vagrant.html) builder.
+
 You'll need to be familiar with Vagrant Cloud, have an upgraded account to
 enable box hosting, and be distributing your box via the [shorthand
 name](https://docs.vagrantup.com/v2/cli/box.html) configuration.
@@ -94,12 +96,18 @@ on Vagrant Cloud, as well as authentication and version information.
 -   `box_download_url` (string) - Optional URL for a self-hosted box. If this
     is set the box will not be uploaded to the Vagrant Cloud.
 
-## Use with Vagrant Post-Processor
+## Use with the Vagrant Post-Processor
 
-You'll need to use the Vagrant post-processor before using this post-processor.
+An example configuration is shown below. Note the use of the nested array that
-An example configuration is below. Note the use of a doubly-nested array, which
+wraps both the Vagrant and Vagrant Cloud post-processors within the
-ensures that the Vagrant Cloud post-processor is run after the Vagrant
+post-processor section. Chaining the post-processors together in this way tells
-post-processor.
+Packer that the artifact produced by the Vagrant post-processor should be passed
+directly to the Vagrant Cloud Post-Processor. It also sets the order in which
+the post-processors should run.
+
+Failure to chain the post-processors together in this way will result in the
+wrong artifact being supplied to the Vagrant Cloud post-processor. This will
+likely cause the Vagrant Cloud post-processor to error and fail.
 
 ``` json
 {
@@ -108,6 +116,10 @@ post-processor.
     "version": "1.0.{{timestamp}}"
   },
   "post-processors": [
+    {
+      "type": "shell-local",
+      "inline": ["echo Doing stuff..."]
+    },
     [
       {
         "type": "vagrant",
@@ -125,3 +137,56 @@ post-processor.
   ]
 }
 ```
+
+## Use with the Artifice Post-Processor
+
+An example configuration is shown below. Note the use of the nested array that
+wraps both the Artifice and Vagrant Cloud post-processors within the
+post-processor section. Chaining the post-processors together in this way tells
+Packer that the artifact produced by the Artifice post-processor should be
+passed directly to the Vagrant Cloud Post-Processor. It also sets the order in
+which the post-processors should run.
+
+Failure to chain the post-processors together in this way will result in the
+wrong artifact being supplied to the Vagrant Cloud post-processor. This will
+likely cause the Vagrant Cloud post-processor to error and fail.
+
+Note that the Vagrant box specified in the Artifice post-processor `files` array
+must end in the `.box` extension. It must also be the first file in the array.
+Additional files bundled by the Artifice post-processor will be ignored.
+
+```json
+{
+  "variables": {
+    "cloud_token": "{{ env `VAGRANT_CLOUD_TOKEN` }}",
+  },
+
+  "builders": [
+    {
+      "type": "null",
+      "communicator": "none"
+    }
+  ],
+
+  "post-processors": [
+    {
+      "type": "shell-local",
+      "inline": ["echo Doing stuff..."]
+    },
+    [
+      {
+        "type": "artifice",
+        "files": [
+          "./path/to/my.box"
+        ]
+      },
+      {
+        "type": "vagrant-cloud",
+        "box_tag": "myorganisation/mybox",
+        "access_token": "{{user `cloud_token`}}",
+        "version": "0.1.0",
+      }
+    ]
+  ]
+}
+```