Merge pull request #2814 from mitchellh/b-2793-repro

Add a test case to validate download file size when downloading from docker

builder/docker/communicator_test.go

@@ -5,12 +5,12 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
-	"runtime"
 	"strings"
 	"testing"
 
 	"github.com/mitchellh/packer/packer"
 	"github.com/mitchellh/packer/provisioner/file"
+	"github.com/mitchellh/packer/provisioner/shell"
 	"github.com/mitchellh/packer/template"
 )
 
@@ -18,6 +18,7 @@ func TestCommunicator_impl(t *testing.T) {
 	var _ packer.Communicator = new(Communicator)
 }
 
+// TestUploadDownload verifies that basic upload / download functionality works
 func TestUploadDownload(t *testing.T) {
 	ui := packer.TestUi(t)
 	cache := &packer.FileCache{CacheDir: os.TempDir()}
@@ -27,13 +28,9 @@ func TestUploadDownload(t *testing.T) {
 		t.Fatalf("Unable to parse config: %s", err)
 	}
 
-	// Make sure we only run this on linux hosts
 	if os.Getenv("PACKER_ACC") == "" {
 		t.Skip("This test is only run with PACKER_ACC=1")
 	}
-	if runtime.GOOS != "linux" {
-		t.Skip("This test is only supported on linux")
-	}
 	cmd := exec.Command("docker", "-v")
 	cmd.Run()
 	if !cmd.ProcessState.Success() {
@@ -102,13 +99,121 @@ func TestUploadDownload(t *testing.T) {
 	}
 }
 
+// TestLargeDownload verifies that files are the apporpriate size after being
+// downloaded. This is to identify and fix the race condition in #2793. You may
+// need to use github.com/cbednarski/rerun to verify since this problem occurs
+// only intermittently.
+func TestLargeDownload(t *testing.T) {
+	ui := packer.TestUi(t)
+	cache := &packer.FileCache{CacheDir: os.TempDir()}
+
+	tpl, err := template.Parse(strings.NewReader(dockerLargeBuilderConfig))
+	if err != nil {
+		t.Fatalf("Unable to parse config: %s", err)
+	}
+
+	if os.Getenv("PACKER_ACC") == "" {
+		t.Skip("This test is only run with PACKER_ACC=1")
+	}
+	cmd := exec.Command("docker", "-v")
+	cmd.Run()
+	if !cmd.ProcessState.Success() {
+		t.Error("docker command not found; please make sure docker is installed")
+	}
+
+	// Setup the builder
+	builder := &Builder{}
+	warnings, err := builder.Prepare(tpl.Builders["docker"].Config)
+	if err != nil {
+		t.Fatalf("Error preparing configuration %s", err)
+	}
+	if len(warnings) > 0 {
+		t.Fatal("Encountered configuration warnings; aborting")
+	}
+
+	// Setup the provisioners
+	shell := &shell.Provisioner{}
+	err = shell.Prepare(tpl.Provisioners[0].Config)
+	if err != nil {
+		t.Fatalf("Error preparing shell provisioner: %s", err)
+	}
+	downloadCupcake := &file.Provisioner{}
+	err = downloadCupcake.Prepare(tpl.Provisioners[1].Config)
+	if err != nil {
+		t.Fatalf("Error preparing downloadCupcake: %s", err)
+	}
+	downloadBigcake := &file.Provisioner{}
+	err = downloadBigcake.Prepare(tpl.Provisioners[2].Config)
+	if err != nil {
+		t.Fatalf("Error preparing downloadBigcake: %s", err)
+	}
+
+	// Preemptive cleanup.
+	defer os.Remove("cupcake")
+	defer os.Remove("bigcake")
+
+	// Add hooks so the provisioners run during the build
+	hooks := map[string][]packer.Hook{}
+	hooks[packer.HookProvision] = []packer.Hook{
+		&packer.ProvisionHook{
+			Provisioners: []packer.Provisioner{
+				shell,
+				downloadCupcake,
+				downloadBigcake,
+			},
+		},
+	}
+	hook := &packer.DispatchHook{Mapping: hooks}
+
+	// Run things
+	artifact, err := builder.Run(ui, hook, cache)
+	if err != nil {
+		t.Fatalf("Error running build %s", err)
+	}
+	// Preemptive cleanup
+	defer artifact.Destroy()
+
+	// Verify that the things we downloaded are the right size. Complain loudly
+	// if they are not.
+	//
+	// cupcake should be 2097152 bytes
+	// bigcake should be 104857600 bytes
+	cupcake, err := os.Stat("cupcake")
+	if err != nil {
+		t.Fatalf("Unable to stat cupcake file: %s", err)
+	}
+	cupcakeExpected := int64(2097152)
+	if cupcake.Size() != cupcakeExpected {
+		t.Errorf("Expected cupcake to be %d bytes; found %d", cupcakeExpected, cupcake.Size())
+	}
+
+	bigcake, err := os.Stat("bigcake")
+	if err != nil {
+		t.Fatalf("Unable to stat bigcake file: %s", err)
+	}
+	bigcakeExpected := int64(104857600)
+	if bigcake.Size() != bigcakeExpected {
+		t.Errorf("Expected bigcake to be %d bytes; found %d", bigcakeExpected, bigcake.Size())
+	}
+
+	// TODO if we can, calculate a sha inside the container and compare to the
+	// one we get after we pull it down. We will probably have to parse the log
+	// or ui output to do this because we use /dev/urandom to create the file.
+
+	// if sha256.Sum256(inputFile) != sha256.Sum256(outputFile) {
+	// 	t.Fatalf("Input and output files do not match\n"+
+	// 		"Input:\n%s\nOutput:\n%s\n", inputFile, outputFile)
+	// }
+
+}
+
 const dockerBuilderConfig = `
 {
   "builders": [
     {
       "type": "docker",
-      "image": "alpine",
-      "export_path": "alpine.tar",
+      "image": "ubuntu",
+      "discard": true,
       "run_command": ["-d", "-i", "-t", "{{.Image}}", "/bin/sh"]
     }
   ],
@@ -127,3 +232,38 @@ const dockerBuilderConfig = `
   ]
 }
 `
+
+const dockerLargeBuilderConfig = `
+{
+  "builders": [
+    {
+      "type": "docker",
+      "image": "ubuntu",
+      "discard": true
+    }
+  ],
+  "provisioners": [
+    {
+      "type": "shell",
+      "inline": [
+        "dd if=/dev/urandom of=/tmp/cupcake bs=1M count=2",
+        "dd if=/dev/urandom of=/tmp/bigcake bs=1M count=100",
+        "sync",
+        "md5sum /tmp/cupcake /tmp/bigcake"
+      ]
+    },
+    {
+      "type": "file",
+      "source": "/tmp/cupcake",
+      "destination": "cupcake",
+      "direction": "download"
+    },
+    {
+      "type": "file",
+      "source": "/tmp/bigcake",
+      "destination": "bigcake",
+      "direction": "download"
+    }
+  ]
+}
+`