package common import ( "context" "fmt" "math/rand" "time" ) const ( // UnlimitedNumAttemptsValue is the value for indicating unlimited attempts for reaching success UnlimitedNumAttemptsValue = uint(0) // number of characters contained in the generated retry token generatedRetryTokenLength = 30 ) // OCIRetryableRequest represents a request that can be reissued according to the specified policy. type OCIRetryableRequest interface { // Any retryable request must implement the OCIRequest interface OCIRequest // Each operation specifies default retry behavior. By passing no arguments to this method, the default retry // behavior, as determined on a per-operation-basis, will be honored. Variadic retry policy option arguments // passed to this method will override the default behavior. RetryPolicy() *RetryPolicy } // OCIOperationResponse represents the output of an OCIOperation, with additional context of error message // and operation attempt number. type OCIOperationResponse struct { // Response from OCI Operation Response OCIResponse // Error from OCI Operation Error error // Operation Attempt Number (one-based) AttemptNumber uint } // NewOCIOperationResponse assembles an OCI Operation Response object. func NewOCIOperationResponse(response OCIResponse, err error, attempt uint) OCIOperationResponse { return OCIOperationResponse{ Response: response, Error: err, AttemptNumber: attempt, } } // RetryPolicy is the class that holds all relevant information for retrying operations. type RetryPolicy struct { // MaximumNumberAttempts is the maximum number of times to retry a request. Zero indicates an unlimited // number of attempts. MaximumNumberAttempts uint // ShouldRetryOperation inspects the http response, error, and operation attempt number, and // - returns true if we should retry the operation // - returns false otherwise ShouldRetryOperation func(OCIOperationResponse) bool // GetNextDuration computes the duration to pause between operation retries. NextDuration func(OCIOperationResponse) time.Duration } // NoRetryPolicy is a helper method that assembles and returns a return policy that indicates an operation should // never be retried (the operation is performed exactly once). func NoRetryPolicy() RetryPolicy { dontRetryOperation := func(OCIOperationResponse) bool { return false } zeroNextDuration := func(OCIOperationResponse) time.Duration { return 0 * time.Second } return NewRetryPolicy(uint(1), dontRetryOperation, zeroNextDuration) } // NewRetryPolicy is a helper method for assembling a Retry Policy object. func NewRetryPolicy(attempts uint, retryOperation func(OCIOperationResponse) bool, nextDuration func(OCIOperationResponse) time.Duration) RetryPolicy { return RetryPolicy{ MaximumNumberAttempts: attempts, ShouldRetryOperation: retryOperation, NextDuration: nextDuration, } } // shouldContinueIssuingRequests returns true if we should continue retrying a request, based on the current attempt // number and the maximum number of attempts specified, or false otherwise. func shouldContinueIssuingRequests(current, maximum uint) bool { return maximum == UnlimitedNumAttemptsValue || current <= maximum } // generateRetryToken generates a retry token that must be included on any request passed to the Retry method. func generateRetryToken() string { alphanumericChars := []rune("abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ") retryToken := make([]rune, generatedRetryTokenLength) for i := range retryToken { retryToken[i] = alphanumericChars[rand.Intn(len(alphanumericChars))] } return string(retryToken) } // Retry is a package-level operation that executes the retryable request using the specified operation and retry policy. func Retry(ctx context.Context, request OCIRetryableRequest, operation OCIOperation, policy RetryPolicy) (OCIResponse, error) { // use a one-based counter because it's easier to think about operation retry in terms of attempt numbering for currentOperationAttempt := uint(1); shouldContinueIssuingRequests(currentOperationAttempt, policy.MaximumNumberAttempts); currentOperationAttempt++ { Debugln(fmt.Sprintf("operation attempt #%v", currentOperationAttempt)) response, err := operation(ctx, request) operationResponse := NewOCIOperationResponse(response, err, currentOperationAttempt) select { case <-ctx.Done(): // return why the request was aborted (could be user interrupted or deadline exceeded) // => include last received response for information (user may choose to re-issue request) return response, ctx.Err() default: // non-blocking select } if policy.ShouldRetryOperation(operationResponse) { // this conditional is explicitly not added to the encompassing if condition to retry based on response // => it is only to determine if, on the last round of this loop, we still skip sleeping (if we're the // last attempt, then there's no point sleeping before we round the loop again and fall out to the // Maximum Number Attempts exceeded error) if currentOperationAttempt != policy.MaximumNumberAttempts { // sleep before retrying the operation duration := policy.NextDuration(operationResponse) if deadline, ok := ctx.Deadline(); ok && time.Now().Add(duration).After(deadline) { // we want to retry the operation, but the policy is telling us to wait for a duration that exceeds // the specified overall deadline for the operation => instead of waiting for however long that // time period is and then aborting, abort now and save the cycles return response, DeadlineExceededByBackoff } Debugln(fmt.Sprintf("waiting %v before retrying operation", duration)) time.Sleep(duration) } } else { // we should NOT retry operation based on response and/or error => return return response, err } } return nil, fmt.Errorf("maximum number of attempts exceeded (%v)", policy.MaximumNumberAttempts) }