2018-07-16 11:49:08 -04:00
|
|
|
package common
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"fmt"
|
|
|
|
"math/rand"
|
2020-04-22 09:28:52 -04:00
|
|
|
"runtime"
|
2018-07-16 11:49:08 -04:00
|
|
|
"time"
|
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
|
|
|
// UnlimitedNumAttemptsValue is the value for indicating unlimited attempts for reaching success
|
|
|
|
UnlimitedNumAttemptsValue = uint(0)
|
|
|
|
|
|
|
|
// number of characters contained in the generated retry token
|
2020-04-22 09:28:52 -04:00
|
|
|
generatedRetryTokenLength = 32
|
2018-07-16 11:49:08 -04:00
|
|
|
)
|
|
|
|
|
|
|
|
// OCIRetryableRequest represents a request that can be reissued according to the specified policy.
|
|
|
|
type OCIRetryableRequest interface {
|
|
|
|
// Any retryable request must implement the OCIRequest interface
|
|
|
|
OCIRequest
|
|
|
|
|
|
|
|
// Each operation specifies default retry behavior. By passing no arguments to this method, the default retry
|
|
|
|
// behavior, as determined on a per-operation-basis, will be honored. Variadic retry policy option arguments
|
|
|
|
// passed to this method will override the default behavior.
|
|
|
|
RetryPolicy() *RetryPolicy
|
|
|
|
}
|
|
|
|
|
|
|
|
// OCIOperationResponse represents the output of an OCIOperation, with additional context of error message
|
|
|
|
// and operation attempt number.
|
|
|
|
type OCIOperationResponse struct {
|
|
|
|
// Response from OCI Operation
|
|
|
|
Response OCIResponse
|
|
|
|
|
|
|
|
// Error from OCI Operation
|
|
|
|
Error error
|
|
|
|
|
|
|
|
// Operation Attempt Number (one-based)
|
|
|
|
AttemptNumber uint
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewOCIOperationResponse assembles an OCI Operation Response object.
|
|
|
|
func NewOCIOperationResponse(response OCIResponse, err error, attempt uint) OCIOperationResponse {
|
|
|
|
return OCIOperationResponse{
|
|
|
|
Response: response,
|
|
|
|
Error: err,
|
|
|
|
AttemptNumber: attempt,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// RetryPolicy is the class that holds all relevant information for retrying operations.
|
|
|
|
type RetryPolicy struct {
|
|
|
|
// MaximumNumberAttempts is the maximum number of times to retry a request. Zero indicates an unlimited
|
|
|
|
// number of attempts.
|
|
|
|
MaximumNumberAttempts uint
|
|
|
|
|
|
|
|
// ShouldRetryOperation inspects the http response, error, and operation attempt number, and
|
|
|
|
// - returns true if we should retry the operation
|
|
|
|
// - returns false otherwise
|
|
|
|
ShouldRetryOperation func(OCIOperationResponse) bool
|
|
|
|
|
|
|
|
// GetNextDuration computes the duration to pause between operation retries.
|
|
|
|
NextDuration func(OCIOperationResponse) time.Duration
|
|
|
|
}
|
|
|
|
|
|
|
|
// NoRetryPolicy is a helper method that assembles and returns a return policy that indicates an operation should
|
|
|
|
// never be retried (the operation is performed exactly once).
|
|
|
|
func NoRetryPolicy() RetryPolicy {
|
|
|
|
dontRetryOperation := func(OCIOperationResponse) bool { return false }
|
|
|
|
zeroNextDuration := func(OCIOperationResponse) time.Duration { return 0 * time.Second }
|
|
|
|
return NewRetryPolicy(uint(1), dontRetryOperation, zeroNextDuration)
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewRetryPolicy is a helper method for assembling a Retry Policy object.
|
|
|
|
func NewRetryPolicy(attempts uint, retryOperation func(OCIOperationResponse) bool, nextDuration func(OCIOperationResponse) time.Duration) RetryPolicy {
|
|
|
|
return RetryPolicy{
|
|
|
|
MaximumNumberAttempts: attempts,
|
|
|
|
ShouldRetryOperation: retryOperation,
|
|
|
|
NextDuration: nextDuration,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// shouldContinueIssuingRequests returns true if we should continue retrying a request, based on the current attempt
|
|
|
|
// number and the maximum number of attempts specified, or false otherwise.
|
|
|
|
func shouldContinueIssuingRequests(current, maximum uint) bool {
|
|
|
|
return maximum == UnlimitedNumAttemptsValue || current <= maximum
|
|
|
|
}
|
|
|
|
|
2020-04-22 09:28:52 -04:00
|
|
|
// RetryToken generates a retry token that must be included on any request passed to the Retry method.
|
|
|
|
func RetryToken() string {
|
2018-07-16 11:49:08 -04:00
|
|
|
alphanumericChars := []rune("abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ")
|
|
|
|
retryToken := make([]rune, generatedRetryTokenLength)
|
|
|
|
for i := range retryToken {
|
|
|
|
retryToken[i] = alphanumericChars[rand.Intn(len(alphanumericChars))]
|
|
|
|
}
|
|
|
|
return string(retryToken)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Retry is a package-level operation that executes the retryable request using the specified operation and retry policy.
|
|
|
|
func Retry(ctx context.Context, request OCIRetryableRequest, operation OCIOperation, policy RetryPolicy) (OCIResponse, error) {
|
|
|
|
|
2020-04-22 09:28:52 -04:00
|
|
|
type retrierResult struct {
|
|
|
|
response OCIResponse
|
|
|
|
err error
|
|
|
|
}
|
|
|
|
|
|
|
|
var response OCIResponse
|
|
|
|
var err error
|
|
|
|
retrierChannel := make(chan retrierResult)
|
|
|
|
|
|
|
|
go func() {
|
|
|
|
|
|
|
|
// Deal with panics more graciously
|
|
|
|
defer func() {
|
|
|
|
if r := recover(); r != nil {
|
|
|
|
stackBuffer := make([]byte, 1024)
|
|
|
|
bytesWritten := runtime.Stack(stackBuffer, false)
|
|
|
|
stack := string(stackBuffer[:bytesWritten])
|
|
|
|
retrierChannel <- retrierResult{nil, fmt.Errorf("panicked while retrying operation. Panic was: %s\nStack: %s", r, stack)}
|
2018-07-16 11:49:08 -04:00
|
|
|
}
|
2020-04-22 09:28:52 -04:00
|
|
|
}()
|
|
|
|
|
|
|
|
// use a one-based counter because it's easier to think about operation retry in terms of attempt numbering
|
|
|
|
for currentOperationAttempt := uint(1); shouldContinueIssuingRequests(currentOperationAttempt, policy.MaximumNumberAttempts); currentOperationAttempt++ {
|
|
|
|
Debugln(fmt.Sprintf("operation attempt #%v", currentOperationAttempt))
|
|
|
|
response, err = operation(ctx, request)
|
|
|
|
operationResponse := NewOCIOperationResponse(response, err, currentOperationAttempt)
|
|
|
|
|
|
|
|
if !policy.ShouldRetryOperation(operationResponse) {
|
|
|
|
// we should NOT retry operation based on response and/or error => return
|
|
|
|
retrierChannel <- retrierResult{response, err}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
duration := policy.NextDuration(operationResponse)
|
|
|
|
//The following condition is kept for backwards compatibility reasons
|
|
|
|
if deadline, ok := ctx.Deadline(); ok && time.Now().Add(duration).After(deadline) {
|
|
|
|
// we want to retry the operation, but the policy is telling us to wait for a duration that exceeds
|
|
|
|
// the specified overall deadline for the operation => instead of waiting for however long that
|
|
|
|
// time period is and then aborting, abort now and save the cycles
|
|
|
|
retrierChannel <- retrierResult{response, DeadlineExceededByBackoff}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
Debugln(fmt.Sprintf("waiting %v before retrying operation", duration))
|
|
|
|
// sleep before retrying the operation
|
|
|
|
<-time.After(duration)
|
2018-07-16 11:49:08 -04:00
|
|
|
}
|
2020-04-22 09:28:52 -04:00
|
|
|
|
|
|
|
retrierChannel <- retrierResult{response, err}
|
|
|
|
}()
|
|
|
|
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
return response, ctx.Err()
|
|
|
|
case result := <-retrierChannel:
|
|
|
|
return result.response, result.err
|
2018-07-16 11:49:08 -04:00
|
|
|
}
|
|
|
|
}
|