From 14166fdd99fb21ad13084081d9518c6e2c780cd5 Mon Sep 17 00:00:00 2001 From: Megan Marsh Date: Thu, 7 Jun 2018 09:30:49 -0700 Subject: [PATCH] update amazon import docs to include the env vars for setting aws waiter delays and timeouts --- builder/amazon/common/state.go | 8 +++ builder/amazon/common/state_test.go | 63 +++++++++++++++++++ website/source/docs/builders/amazon.html.md | 19 +++++- .../post-processors/amazon-import.html.md | 53 +++++++++------- 4 files changed, 121 insertions(+), 22 deletions(-) create mode 100644 builder/amazon/common/state_test.go diff --git a/builder/amazon/common/state.go b/builder/amazon/common/state.go index 8d5896c70..741b5c651 100644 --- a/builder/amazon/common/state.go +++ b/builder/amazon/common/state.go @@ -278,6 +278,14 @@ func getWaiterOptions() []request.WaiterOption { } waitOpts = append(waitOpts, request.WithWaiterMaxAttempts(maxAttempts)) } + if len(waitOpts) == 0 { + log.Printf("No AWS timeout and polling overrides have been set. " + + "Packer will defalt to waiter-specific delays and timeouts. If you would " + + "like to customize the length of time between retries and max " + + "number of retries you may do so by setting the environment " + + "variables AWS_POLL_DELAY_SECONDS and AWS_MAX_ATTEMPTS to your " + + "desired values.") + } return waitOpts } diff --git a/builder/amazon/common/state_test.go b/builder/amazon/common/state_test.go new file mode 100644 index 000000000..638a41ae9 --- /dev/null +++ b/builder/amazon/common/state_test.go @@ -0,0 +1,63 @@ +package common + +import ( + "os" + "reflect" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws/request" +) + +func clearEnvVars() { + os.Unsetenv("AWS_POLL_DELAY_SECONDS") + os.Unsetenv("AWS_MAX_ATTEMPTS") + os.Unsetenv("AWS_TIMEOUT_SECONDS") +} + +func testGetWaiterOptions(t *testing.T) { + clearEnvVars() + + // no vars are set + options := getWaiterOptions() + if len(options) > 0 { + t.Fatalf("Did not expect any waiter options to be generated; actual: %#v", options) + } + + // all vars are set + os.Setenv("AWS_MAX_ATTEMPTS", "800") + os.Setenv("AWS_TIMEOUT_SECONDS", "20") + os.Setenv("AWS_POLL_DELAY_SECONDS", "1") + options = getWaiterOptions() + expected := []request.WaiterOption{ + request.WithWaiterDelay(request.ConstantWaiterDelay(time.Duration(1) * time.Second)), + request.WithWaiterMaxAttempts(800), + } + if !reflect.DeepEqual(options, expected) { + t.Fatalf("expected != actual!! Expected: %#v; Actual: %#v.", expected, options) + } + clearEnvVars() + + // poll delay is not set + os.Setenv("AWS_MAX_ATTEMPTS", "800") + options = getWaiterOptions() + expected = []request.WaiterOption{ + request.WithWaiterMaxAttempts(800), + } + if !reflect.DeepEqual(options, expected) { + t.Fatalf("expected != actual!! Expected: %#v; Actual: %#v.", expected, options) + } + clearEnvVars() + + // poll delay is not set but timeout seconds is + os.Setenv("AWS_TIMEOUT_SECONDS", "20") + options = getWaiterOptions() + expected = []request.WaiterOption{ + request.WithWaiterDelay(request.ConstantWaiterDelay(time.Duration(2) * time.Second)), + request.WithWaiterMaxAttempts(10), + } + if !reflect.DeepEqual(options, expected) { + t.Fatalf("expected != actual!! Expected: %#v; Actual: %#v.", expected, options) + } + clearEnvVars() +} diff --git a/website/source/docs/builders/amazon.html.md b/website/source/docs/builders/amazon.html.md index 3b1a177a9..b1119f607 100644 --- a/website/source/docs/builders/amazon.html.md +++ b/website/source/docs/builders/amazon.html.md @@ -176,7 +176,7 @@ for Packer to work: "Resource" : "*" }] } -``` +``` Note that if you'd like to create a spot instance, you must also add: @@ -231,3 +231,20 @@ If you suspect your system's date is wrong, you can compare it against . On Linux/OS X, you can run the `date` command to get the current time. If you're on Linux, you can try setting the time with ntp by running `sudo ntpd -q`. + +### `exceeded wait attempts` while waiting for tasks to complete +We use the AWS SDK's built-in waiters to wait for longer-running tasks to +complete. These waiters have default delays between queries and maximum number +of queries that don't always work for our users. + +If you find that you are being rate-limited or have exceeded your max wait +attempts, you can override the defaults by setting the following packer +environment variables (note that these will apply to all aws tasks that we have +to wait for): + +`AWS_MAX_ATTEMPTS` - This is how many times to re-send a status update request. +Excepting tasks that we know can take an extremely long time, this defaults to +40tries. + +`AWS_POLL_DELAY_SECONDS` - How many seconds to wait in between status update +requests. Generally defaults to 2 or 5 seconds, depending on the task. \ No newline at end of file diff --git a/website/source/docs/post-processors/amazon-import.html.md b/website/source/docs/post-processors/amazon-import.html.md index f7f8959a9..836a271c3 100644 --- a/website/source/docs/post-processors/amazon-import.html.md +++ b/website/source/docs/post-processors/amazon-import.html.md @@ -125,31 +125,42 @@ This is an example that uses `vmware-iso` builder and exports the `.ova` file us ``` json "post-processors" : [ [ - { - "type": "shell-local", - "inline": [ "/usr/bin/ovftool /.vmx /.ova" ] - }, - { - "files": [ - "/.ova" - ], - "type": "artifice" - }, - { - "type": "amazon-import", - "access_key": "YOUR KEY HERE", - "secret_key": "YOUR SECRET KEY HERE", - "region": "us-east-1", - "s3_bucket_name": "importbucket", - "license_type": "BYOL", - "tags": { - "Description": "packer amazon-import {{timestamp}}" - } - } + { + "type": "shell-local", + "inline": [ "/usr/bin/ovftool /.vmx /.ova" ] + }, + { + "files": [ + "/.ova" + ], + "type": "artifice" + }, + { + "type": "amazon-import", + "access_key": "YOUR KEY HERE", + "secret_key": "YOUR SECRET KEY HERE", + "region": "us-east-1", + "s3_bucket_name": "importbucket", + "license_type": "BYOL", + "tags": { + "Description": "packer amazon-import {{timestamp}}" + } + } ] ] ``` +## Troubleshooting Timeouts +The amazon-import feature can take a long time to upload and convert your OVAs +into AMIs; if you find that your build is failing because you have exceeded your +max retries or find yourself being rate limited, you can override the max +retries and the delay in between retries by setting the environment variables + `AWS_MAX_ATTEMPTS` and `AWS_POLL_DELAY_SECONDS` on the machine running the + Packer build. By default, the waiter that waits for your image to be imported + from s3 waits retries up to 300 times with a 5 second delay in between retries. + This is dramatically higher than many of our other waiters, to account for how + long this process can take. + -> **Note:** Packer can also read the access key and secret access key from environmental variables. See the configuration reference in the section above for more information on what environmental variables Packer will look for.