From 939b5ee22d907e002f22639ebc162779955c2610 Mon Sep 17 00:00:00 2001 From: Moss Date: Mon, 6 Apr 2020 18:06:46 +0200 Subject: [PATCH 1/4] Update CONTRIBUTING with provisioner acc tests --- .github/CONTRIBUTING.md | 193 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 193 insertions(+) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 1d08d09a0..ae1f19343 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -328,6 +328,199 @@ Acceptance tests typically require other environment variables to be set for things such as API tokens and keys. Each test should error and tell you which credentials are missing, so those are not documented here. +#### Running Provisioners Acceptance Tests + +To run the Provisioners Acceptance Tests you should set both ACC_TEST_BUILDERS and ACC_TEST_PROVISIONERS environment variables. +These will tell which provisioner and builder should run the test against with. + +Examples of usage: + +``` +ACC_TEST_BUILDERS=amazon-ebs ACC_TEST_PROVISIONERS=shell go test ./provisioner/shell/... -v -timeout=1 +``` +Will run the Shell provisioner acceptance tests against the Amazon EBS builder. +``` +ACC_TEST_BUILDERS=amazon-ebs ACC_TEST_PROVISIONERS=shell make provisioners-acctest +``` +Will do the same but using the Makefile +``` +ACC_TEST_BUILDERS=amazon-ebs ACC_TEST_PROVISIONERS=shell,powershell make provisioners-acctest +``` +Will run the all Shell and Powershell provisioners acceptance tests against the Amazon EBS builder. +``` +ACC_TEST_BUILDERS=amazon-ebs ACC_TEST_PROVISIONERS=all make provisioners-acctest +``` +Will run the all provisioners acceptance tests against the Amazon EBS builder. +``` +ACC_TEST_BUILDERS=all ACC_TEST_PROVISIONERS=all make provisioners-acctest +``` +Will run the all provisioners acceptance tests against all builders whenever they are compatible. + +Both ACC_TEST_BUILDERS and ACC_TEST_PROVISIONERS allows defining a list of builders and provisioners separated by comma +(e.g. ACC_TEST_BUILDERS=amazon-ebs,virtualbox-iso) + + +#### Writing Provisioner Acceptance Tests + +Packer has an already implemented structure that will run the provisioner against builders and you can find it in `helper/tests/acc/provisioners.go`. +All provisioner's acceptance test should use this structure in their acceptance tests. + +To start writing a new provisioner acceptance test, you should add a test file named as `provisioner_acc_test.go` in the provisioner folder +and the package should be `_test`. This file should have a struct that will implement the ProvisionerAcceptance interface. + +```go +type ProvisionerAcceptance interface { + GetName() string + GetConfig() (string, error) + GetProvisionerStore() packer.MapOfProvisioner + IsCompatible(builder string, vmOS string) bool + RunTest(c *command.BuildCommand, args []string) error +} +``` + +- **GetName()** should return the provisioner type. For example for the Shell provisioner the method returns "shell". + +- **GetConfig()** should read a text file with the json configuration block for the provisioner and any other necessary. +For the Shell provisioner the file contains: + + ``` + { + "type": "shell", + "inline": [ + "echo {{ build `ID`}} > provisioner.{{ build `PackerRunUUID`}}.txt" + ] + }, + { + "type": "file", + "source": "provisioner.{{ build `PackerRunUUID`}}.txt", + "destination": "provisioner.shell.{{ build `PackerRunUUID`}}.txt", + "direction": "download" + } + ``` + The file should be placed under the `test-fixtures` folder. + In this case, it's necessary to use the File provisioner to validate if the Shell provisioner test is successful or not. + This config should be returned as string that will be later merged with the builder config into a full template. + +- **GetProvisionerStore()** this returns the provisioner store where we declare the available provisioners for running the build. +For Shell provisioners this is: + ```go + func (s *ShellProvisionerAccTest) GetProvisionerStore() packer.MapOfProvisioner { + return packer.MapOfProvisioner{ + "shell": func() (packer.Provisioner, error) { return &shell.Provisioner{}, nil }, + "file": func() (packer.Provisioner, error) { return &file.Provisioner{}, nil }, + } + } + ``` + +- **IsCompatible(builder string, vmOS string)** returns true or false whether the provisioner should run against a +specific builder or/and specific OS. + +- **RunTest(c \*command.BuildCommand, args []string)** it will actually run the build and return any error if it fails any necessary validation. +For Shell provisioner this is: + ```go + func (s *ShellProvisionerAccTest) RunTest(c *command.BuildCommand, args []string) error { + // Provisioner specific setup + UUID := os.Getenv("PACKER_RUN_UUID") + if UUID == "" { + UUID, _ = uuid.GenerateUUID() + os.Setenv("PACKER_RUN_UUID", UUID) + } + file := "provisioner.shell." + UUID + ".txt" + defer testshelper.CleanupFiles(file) + + // Run build + // All provisioner acc tests should contain this code and validation + if code := c.Run(args); code != 0 { + ui := c.Meta.Ui.(*packer.BasicUi) + out := ui.Writer.(*bytes.Buffer) + err := ui.ErrorWriter.(*bytes.Buffer) + return fmt.Errorf( + "Bad exit code.\n\nStdout:\n\n%s\n\nStderr:\n\n%s", + out.String(), + err.String()) + } + + // Any other extra specific validation + if !testshelper.FileExists(file) { + return fmt.Errorf("Expected to find %s", file) + } + return nil + } + + ``` + +After writing the struct and implementing the interface, now is time to actually write the test that will run all +of this code. Your test should look like: + +```go +func TestShellProvisioner(t *testing.T) { + p := os.Getenv("ACC_TEST_PROVISIONERS") + if p != "all" && !strings.Contains(p, "shell") { + t.Skip() + } + acc.TestProvisionersAgainstBuilders(new(ShellProvisionerAccTest), t) +} +``` + +If the environment variable **ACC_TEST_PROVISIONERS** is set as `all` or contains the provisioner type the test should run, otherwise the test should skip. +In case of running it, you'll need to call the helper function `acc.TestProvisionersAgainstBuilders` passing a pointer to the test struct created above and the test testing pointer. + +The method `TestProvisionersAgainstBuilders` will run the provisioner against all available and compatible builder. An available builder +is the one that has the necessary code for running this type of test. In case the builder you want to run against is not available for testing, you can write it following the next steps. + +To add a new builder to the available builders for acc testing, you'll need to create a new folder under the builder folder +called `acceptace` and inside you create the `builder_acceptance.go` file and the package should be `_acc`. Like the provisioners, you'll need to create a struct that will +implement the BuilderAcceptance interface. +```go +type BuilderAcceptance interface { + GetConfigs() (map[string]string, error) + GetBuilderStore() packer.MapOfBuilder + CleanUp() error +} +``` +- **GetConfigs()** should read a text file with the json configuration block for the builder and return a map of configs by OS type. +For the Amazon EBS builder the file contains: + ``` + { + "type": "amazon-ebs", + "ami_name": "packer-acc-test", + "instance_type": "m1.small", + "region": "us-east-1", + "ssh_username": "ubuntu", + "source_ami": "ami-0568456c", + "force_deregister" : true, + "tags": { + "packer-test": "true" + } + } + ``` + The file should be placed under the `test-fixtures` folder. + In case you need to make references to another file, you'll need to add the relative path to provisioners folder like: + `../../builder/amazon/ebs/acceptance/test-fixtures/file.txt`. + +- **GetBuilderStore()** this returns the builder store where we declare the available builders for running the build. +For the Amazon EBS builder this is: + ```go + func (s *AmazonEBSAccTest) GetBuilderStore() packer.MapOfBuilder { + return packer.MapOfBuilder{ + "amazon-ebs": func() (packer.Builder, error) { return &amazonebsbuilder.Builder{}, nil }, + } + } + ``` + +- **CleanUp()** cleans any resource created by the builder whether local or remote. + +Once you created the builder necessary code, the last step is adding it to the `BuildersAccTest` map in `helper/tests/acc/provisioners.go`. +```go +var BuildersAccTest = map[string]BuilderAcceptance{ + ... + "amazon-ebs": new(amazonEBS.AmazonEBSAccTest), + ... +} +``` + +Once you finish the steps, you should be ready to run your new provisioner acceptance test. + #### Debugging Plugins Each packer plugin runs in a separate process and communicates via RPC over a From 25b91aa3cbb60439e11ba3dca7d3ae07677775cb Mon Sep 17 00:00:00 2001 From: Moss Date: Mon, 6 Apr 2020 18:22:34 +0200 Subject: [PATCH 2/4] Update CONTRIBUTING with ajustments --- .github/CONTRIBUTING.md | 77 +++++++++++++++++++++++------------------ 1 file changed, 43 insertions(+), 34 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index ae1f19343..96e5c6c6e 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -330,40 +330,49 @@ credentials are missing, so those are not documented here. #### Running Provisioners Acceptance Tests -To run the Provisioners Acceptance Tests you should set both ACC_TEST_BUILDERS and ACC_TEST_PROVISIONERS environment variables. -These will tell which provisioner and builder should run the test against with. +**Warning:** The acceptance tests create/destroy/modify _real resources_, which +may incur costs for real money. In the presence of a bug, it is possible that +resources may be left behind, which can cost money even though you were not +using them. We recommend running tests in an account used only for that purpose +so it is easy to see if there are any dangling resources, and so production +resources are not accidentally destroyed or overwritten during testing. +Also, these typically require an API key (AWS, GCE), or additional software +to be installed on your computer (VirtualBox, VMware). + +To run the Provisioners Acceptance Tests you should use both ACC_TEST_BUILDERS and ACC_TEST_PROVISIONERS variables to +tell which provisioner and builder the test should be run against. Examples of usage: -``` -ACC_TEST_BUILDERS=amazon-ebs ACC_TEST_PROVISIONERS=shell go test ./provisioner/shell/... -v -timeout=1 -``` -Will run the Shell provisioner acceptance tests against the Amazon EBS builder. -``` -ACC_TEST_BUILDERS=amazon-ebs ACC_TEST_PROVISIONERS=shell make provisioners-acctest -``` -Will do the same but using the Makefile -``` -ACC_TEST_BUILDERS=amazon-ebs ACC_TEST_PROVISIONERS=shell,powershell make provisioners-acctest -``` -Will run the all Shell and Powershell provisioners acceptance tests against the Amazon EBS builder. -``` -ACC_TEST_BUILDERS=amazon-ebs ACC_TEST_PROVISIONERS=all make provisioners-acctest -``` -Will run the all provisioners acceptance tests against the Amazon EBS builder. -``` -ACC_TEST_BUILDERS=all ACC_TEST_PROVISIONERS=all make provisioners-acctest -``` -Will run the all provisioners acceptance tests against all builders whenever they are compatible. +- Run the Shell provisioner acceptance tests against the Amazon EBS builder. + ``` + ACC_TEST_BUILDERS=amazon-ebs ACC_TEST_PROVISIONERS=shell go test ./provisioner/shell/... -v -timeout=1 + ``` +- Do the same but using the Makefile + ``` + ACC_TEST_BUILDERS=amazon-ebs ACC_TEST_PROVISIONERS=shell make provisioners-acctest + ``` +- Run the all Shell and Powershell provisioners acceptance tests against the Amazon EBS builder. + ``` + ACC_TEST_BUILDERS=amazon-ebs ACC_TEST_PROVISIONERS=shell,powershell make provisioners-acctest + ``` +- Run the all provisioners acceptance tests against the Amazon EBS builder. + ``` + ACC_TEST_BUILDERS=amazon-ebs ACC_TEST_PROVISIONERS=all make provisioners-acctest + ``` +- Run the all provisioners acceptance tests against all builders whenever they are compatible. + ``` + ACC_TEST_BUILDERS=all ACC_TEST_PROVISIONERS=all make provisioners-acctest + ``` Both ACC_TEST_BUILDERS and ACC_TEST_PROVISIONERS allows defining a list of builders and provisioners separated by comma -(e.g. ACC_TEST_BUILDERS=amazon-ebs,virtualbox-iso) +(e.g. `ACC_TEST_BUILDERS=amazon-ebs,virtualbox-iso`) #### Writing Provisioner Acceptance Tests Packer has an already implemented structure that will run the provisioner against builders and you can find it in `helper/tests/acc/provisioners.go`. -All provisioner's acceptance test should use this structure in their acceptance tests. +All provisioner's should use this structure in their acceptance tests. To start writing a new provisioner acceptance test, you should add a test file named as `provisioner_acc_test.go` in the provisioner folder and the package should be `_test`. This file should have a struct that will implement the ProvisionerAcceptance interface. @@ -380,8 +389,8 @@ type ProvisionerAcceptance interface { - **GetName()** should return the provisioner type. For example for the Shell provisioner the method returns "shell". -- **GetConfig()** should read a text file with the json configuration block for the provisioner and any other necessary. -For the Shell provisioner the file contains: +- **GetConfig()** should read a text file with the json configuration block for the provisioner and any other necessary provisioner. +For the Shell one the file contains: ``` { @@ -402,7 +411,7 @@ For the Shell provisioner the file contains: This config should be returned as string that will be later merged with the builder config into a full template. - **GetProvisionerStore()** this returns the provisioner store where we declare the available provisioners for running the build. -For Shell provisioners this is: +For the Shell provisioners this is: ```go func (s *ShellProvisionerAccTest) GetProvisionerStore() packer.MapOfProvisioner { return packer.MapOfProvisioner{ @@ -415,8 +424,8 @@ For Shell provisioners this is: - **IsCompatible(builder string, vmOS string)** returns true or false whether the provisioner should run against a specific builder or/and specific OS. -- **RunTest(c \*command.BuildCommand, args []string)** it will actually run the build and return any error if it fails any necessary validation. -For Shell provisioner this is: +- **RunTest(c \*command.BuildCommand, args []string)** it will actually run the build and return any error if it fails the validations. +For the Shell provisioner this is: ```go func (s *ShellProvisionerAccTest) RunTest(c *command.BuildCommand, args []string) error { // Provisioner specific setup @@ -449,8 +458,8 @@ For Shell provisioner this is: ``` -After writing the struct and implementing the interface, now is time to actually write the test that will run all -of this code. Your test should look like: +After writing the struct and implementing the interface, now is time to write the test that will run all +of this code you wrote. Your test should be like: ```go func TestShellProvisioner(t *testing.T) { @@ -462,13 +471,13 @@ func TestShellProvisioner(t *testing.T) { } ``` -If the environment variable **ACC_TEST_PROVISIONERS** is set as `all` or contains the provisioner type the test should run, otherwise the test should skip. +If the environment variable **ACC_TEST_PROVISIONERS** is set as `all` or contains the provisioner type, then the test should run, otherwise the test should skip. In case of running it, you'll need to call the helper function `acc.TestProvisionersAgainstBuilders` passing a pointer to the test struct created above and the test testing pointer. -The method `TestProvisionersAgainstBuilders` will run the provisioner against all available and compatible builder. An available builder +The method `TestProvisionersAgainstBuilders` will run the provisioner against all available and compatible builders. An available builder is the one that has the necessary code for running this type of test. In case the builder you want to run against is not available for testing, you can write it following the next steps. -To add a new builder to the available builders for acc testing, you'll need to create a new folder under the builder folder +To add a new builder to the available builders for provisioners acc testing, you'll need to create a new folder under the builder folder called `acceptace` and inside you create the `builder_acceptance.go` file and the package should be `_acc`. Like the provisioners, you'll need to create a struct that will implement the BuilderAcceptance interface. ```go From 388997cd1c0d7e43d39d1be9698dd7f70d830be5 Mon Sep 17 00:00:00 2001 From: Moss Date: Tue, 7 Apr 2020 10:29:17 +0200 Subject: [PATCH 3/4] Fix some typos --- .github/CONTRIBUTING.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 96e5c6c6e..a9f7a00d7 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -339,14 +339,14 @@ resources are not accidentally destroyed or overwritten during testing. Also, these typically require an API key (AWS, GCE), or additional software to be installed on your computer (VirtualBox, VMware). -To run the Provisioners Acceptance Tests you should use both ACC_TEST_BUILDERS and ACC_TEST_PROVISIONERS variables to +To run the Provisioners Acceptance Tests you should use both **ACC_TEST_BUILDERS** and **ACC_TEST_PROVISIONERS** variables to tell which provisioner and builder the test should be run against. Examples of usage: - Run the Shell provisioner acceptance tests against the Amazon EBS builder. ``` - ACC_TEST_BUILDERS=amazon-ebs ACC_TEST_PROVISIONERS=shell go test ./provisioner/shell/... -v -timeout=1 + ACC_TEST_BUILDERS=amazon-ebs ACC_TEST_PROVISIONERS=shell go test ./provisioner/shell/... -v -timeout=1h ``` - Do the same but using the Makefile ``` @@ -365,14 +365,14 @@ Examples of usage: ACC_TEST_BUILDERS=all ACC_TEST_PROVISIONERS=all make provisioners-acctest ``` -Both ACC_TEST_BUILDERS and ACC_TEST_PROVISIONERS allows defining a list of builders and provisioners separated by comma +Both **ACC_TEST_BUILDERS** and **ACC_TEST_PROVISIONERS** allows defining a list of builders and provisioners separated by comma (e.g. `ACC_TEST_BUILDERS=amazon-ebs,virtualbox-iso`) #### Writing Provisioner Acceptance Tests Packer has an already implemented structure that will run the provisioner against builders and you can find it in `helper/tests/acc/provisioners.go`. -All provisioner's should use this structure in their acceptance tests. +All provisioners should use this structure in their acceptance tests. To start writing a new provisioner acceptance test, you should add a test file named as `provisioner_acc_test.go` in the provisioner folder and the package should be `_test`. This file should have a struct that will implement the ProvisionerAcceptance interface. @@ -478,7 +478,7 @@ The method `TestProvisionersAgainstBuilders` will run the provisioner against al is the one that has the necessary code for running this type of test. In case the builder you want to run against is not available for testing, you can write it following the next steps. To add a new builder to the available builders for provisioners acc testing, you'll need to create a new folder under the builder folder -called `acceptace` and inside you create the `builder_acceptance.go` file and the package should be `_acc`. Like the provisioners, you'll need to create a struct that will +called `acceptance` and inside you create the `builder_acceptance.go` file and the package should be `_acc`. Like the provisioners, you'll need to create a struct that will implement the BuilderAcceptance interface. ```go type BuilderAcceptance interface { From eac5526a281f07dc9eadb813cbf8bad223da6b28 Mon Sep 17 00:00:00 2001 From: Moss Date: Tue, 7 Apr 2020 11:41:03 +0200 Subject: [PATCH 4/4] Update prov test code --- .github/CONTRIBUTING.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index a9f7a00d7..b88ea01b4 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -463,10 +463,7 @@ of this code you wrote. Your test should be like: ```go func TestShellProvisioner(t *testing.T) { - p := os.Getenv("ACC_TEST_PROVISIONERS") - if p != "all" && !strings.Contains(p, "shell") { - t.Skip() - } + acc.TestProvisionersPreCheck("shell", t) acc.TestProvisionersAgainstBuilders(new(ShellProvisionerAccTest), t) } ```