Implements retries and expo backoffs in Go Client.

src/main/client/go.client.ftl

@@ -49,28 +49,82 @@ func NewClient(httpClient *http.Client, baseURL *url.URL, apiKey string) *Fusion
   return c
 }
 
+// NewClientWithRetryPolicy creates a new FusionAuthClient with the provided retry policy
+// if httpClient is nil then a DefaultClient is used
+func NewClientWithRetryPolicy(httpClient *http.Client, baseURL *url.URL, apiKey string, retryPolicy *RetryPolicy) *FusionAuthClient {
+	if httpClient == nil {
+		httpClient = &http.Client{
+			Timeout: 5 * time.Minute,
+		}
+	}
+	c := &FusionAuthClient{
+		HTTPClient:  httpClient,
+		BaseURL:     baseURL,
+		APIKey:      apiKey,
+		RetryPolicy: retryPolicy,
+	}
+
+	return c
+}
+
 // SetTenantId sets the tenantId on the client
 func (c *FusionAuthClient) SetTenantId(tenantId string)  {
   c.TenantId = tenantId
 }
 
+// RetryPolicy configures retry behaviour for the FusionAuth client.
+// A nil RetryPolicy (the default) means no retries are performed.
+type RetryPolicy struct {
+  // MaxRetries is the number of additional attempts after the initial request.
+  // 0 means no retries.
+  MaxRetries int
+  // ShouldRetry decides whether a response warrants a retry.
+  // Receives the HTTP status code and the raw response body.
+  // When nil, RetryOnConflict is used.
+  ShouldRetry func(statusCode int, body []byte) bool
+  // Backoff returns how long to wait before attempt n (1-indexed: first retry = 1).
+  // When nil, retries are issued immediately with no delay.
+  Backoff func(attempt int) time.Duration
+}
+
+// RetryOnConflict retries 409 responses whose body contains a [retryableConflict] error code.
+func RetryOnConflict(statusCode int, body []byte) bool {
+  return statusCode == http.StatusConflict && bytes.Contains(body, []byte("[retryableConflict]"))
+}
+
+// FixedBackoff returns a backoff function that always waits the same duration.
+func FixedBackoff(d time.Duration) func(attempt int) time.Duration {
+  return func(_ int) time.Duration { return d }
+}
+
+// ExponentialBackoff returns a backoff function that doubles the wait on every retry:
+// base, 2*base, 4*base, …
+func ExponentialBackoff(base time.Duration) func(attempt int) time.Duration {
+  return func(attempt int) time.Duration {
+    return base * (1 << uint(attempt-1))
+  }
+}
+
 // FusionAuthClient describes the Go Client for interacting with FusionAuth's RESTful API
 type FusionAuthClient struct {
-  HTTPClient *http.Client
-  BaseURL    *url.URL
-  APIKey     string
-  Debug      bool
-  TenantId   string
+  HTTPClient  *http.Client
+  BaseURL     *url.URL
+  APIKey      string
+  Debug       bool
+  TenantId    string
+  RetryPolicy *RetryPolicy
 }
 
 type restClient struct {
   Body        io.Reader
+  bodyBytes   []byte
   Debug       bool
   ErrorRef    interface{}
   Headers     map[string]string
   HTTPClient  *http.Client
   Method      string
   ResponseRef interface{}
+  RetryPolicy *RetryPolicy
   Uri         *url.URL
 }
 
@@ -85,6 +139,7 @@ func (c *FusionAuthClient) StartAnonymous(responseRef interface{}, errorRef inte
     Headers:     make(map[string]string),
     HTTPClient:  c.HTTPClient,
     ResponseRef: responseRef,
+    RetryPolicy: c.RetryPolicy,
   }
   rc.Uri, _ = url.Parse(c.BaseURL.String())
   if c.TenantId != "" {
@@ -96,34 +151,89 @@ func (c *FusionAuthClient) StartAnonymous(responseRef interface{}, errorRef inte
 }
 
 func (rc *restClient) Do(ctx context.Context) error {
-  req, err := http.NewRequestWithContext(ctx, rc.Method, rc.Uri.String(), rc.Body)
-  if err != nil {
-    return err
-  }
-  for key, val := range rc.Headers {
-    req.Header.Set(key, val)
-  }
-  resp, err := rc.HTTPClient.Do(req)
-  if err != nil {
-    return err
+  // Buffer the request body once so it can be replayed on retries.
+  if rc.Body != nil {
+    b, err := io.ReadAll(rc.Body)
+    if err != nil {
+      return err
+    }
+    rc.bodyBytes = b
+    rc.Body = nil
   }
-  defer resp.Body.Close()
-  if rc.Debug {
-    responseDump, _ := httputil.DumpResponse(resp, true)
-    fmt.Println(string(responseDump))
+
+  maxAttempts := 1
+  if rc.RetryPolicy != nil && rc.RetryPolicy.MaxRetries > 0 {
+    maxAttempts = 1 + rc.RetryPolicy.MaxRetries
   }
-  if resp.StatusCode < 200 || resp.StatusCode > 299 {
-    if err = json.NewDecoder(resp.Body).Decode(rc.ErrorRef); err == io.EOF {
-      err = nil
+
+  for attempt := 0; attempt < maxAttempts; attempt++ {
+    if attempt > 0 {
+      delay := time.Duration(0)
+      if rc.RetryPolicy.Backoff != nil {
+        delay = rc.RetryPolicy.Backoff(attempt)
+      }
+      if delay > 0 {
+        select {
+        case <-ctx.Done():
+          return ctx.Err()
+        case <-time.After(delay):
+        }
+      }
     }
-  } else {
-    rc.ErrorRef = nil
-    if _, ok := rc.ResponseRef.(*BaseHTTPResponse); !ok {
-      err = json.NewDecoder(resp.Body).Decode(rc.ResponseRef)
+
+    var body io.Reader
+    if rc.bodyBytes != nil {
+      body = bytes.NewReader(rc.bodyBytes)
+    }
+    req, err := http.NewRequestWithContext(ctx, rc.Method, rc.Uri.String(), body)
+    if err != nil {
+      return err
+    }
+    for key, val := range rc.Headers {
+      req.Header.Set(key, val)
+    }
+
+    resp, err := rc.HTTPClient.Do(req)
+    if err != nil {
+      return err
+    }
+    respBody, readErr := io.ReadAll(resp.Body)
+    resp.Body.Close()
+    if readErr != nil {
+      return readErr
     }
+
+    if rc.Debug {
+      resp.Body = io.NopCloser(bytes.NewReader(respBody))
+      responseDump, _ := httputil.DumpResponse(resp, true)
+      fmt.Println(string(responseDump))
+    }
+
+    // Check whether this response should trigger a retry.
+    if attempt < maxAttempts-1 && rc.RetryPolicy != nil {
+      shouldRetry := rc.RetryPolicy.ShouldRetry
+      if shouldRetry == nil {
+        shouldRetry = RetryOnConflict
+      }
+      if shouldRetry(resp.StatusCode, respBody) {
+        continue
+      }
+    }
+
+    if resp.StatusCode < 200 || resp.StatusCode > 299 {
+      if err = json.NewDecoder(bytes.NewReader(respBody)).Decode(rc.ErrorRef); err == io.EOF {
+        err = nil
+      }
+    } else {
+      rc.ErrorRef = nil
+      if _, ok := rc.ResponseRef.(*BaseHTTPResponse); !ok {
+        err = json.NewDecoder(bytes.NewReader(respBody)).Decode(rc.ResponseRef)
+      }
+    }
+    rc.ResponseRef.(StatusAble).SetStatus(resp.StatusCode)
+    return err
   }
-  rc.ResponseRef.(StatusAble).SetStatus(resp.StatusCode)
-  return err
+  return nil
 }
 
 func (rc *restClient) WithAuthorization(key string) *restClient {