Add retry + idempotency on trading when try + add more tts

src/Managing.Infrastructure.Web3/Services/Web3ProxyService.cs

@@ -1,4 +1,5 @@
 using System.Collections;
+using System.Net;
 using System.Net.Http.Json;
 using System.Text;
 using System.Text.Json;
@@ -6,7 +7,9 @@ using System.Web;
 using Managing.Application.Abstractions.Services;
 using Managing.Domain.Accounts;
 using Managing.Infrastructure.Evm.Models.Proxy;
+using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using Polly;
 using static Managing.Common.Enums;
 
 namespace Managing.Infrastructure.Evm.Services
@@ -14,6 +17,9 @@ namespace Managing.Infrastructure.Evm.Services
     public class Web3ProxySettings
     {
         public string BaseUrl { get; set; } = "http://localhost:3000";
+        public int MaxRetryAttempts { get; set; } = 3;
+        public int RetryDelayMs { get; set; } = 1000;
+        public int TimeoutSeconds { get; set; } = 30;
     }
 
     public class Web3ProxyService : IWeb3ProxyService
@@ -21,15 +27,98 @@ namespace Managing.Infrastructure.Evm.Services
         private readonly HttpClient _httpClient;
         private readonly Web3ProxySettings _settings;
         private readonly JsonSerializerOptions _jsonOptions;
+        private readonly IAsyncPolicy<HttpResponseMessage> _retryPolicy;
+        private readonly ILogger<Web3ProxyService> _logger;
 
-        public Web3ProxyService(IOptions<Web3ProxySettings> options)
+        public Web3ProxyService(IOptions<Web3ProxySettings> options, ILogger<Web3ProxyService> logger)
         {
-            _httpClient = new HttpClient();
             _settings = options.Value;
+            _logger = logger;
+            
+            _httpClient = new HttpClient();
+            _httpClient.Timeout = TimeSpan.FromSeconds(_settings.TimeoutSeconds);
+            
             _jsonOptions = new JsonSerializerOptions
             {
                 PropertyNamingPolicy = JsonNamingPolicy.CamelCase
             };
+
+            // Configure retry policy
+            _retryPolicy = Policy
+                .Handle<HttpRequestException>()
+                .Or<TaskCanceledException>()
+                .Or<TimeoutException>()
+                .OrResult<HttpResponseMessage>(r => !r.IsSuccessStatusCode && IsRetryableStatusCode(r.StatusCode))
+                .WaitAndRetryAsync(
+                    retryCount: _settings.MaxRetryAttempts,
+                    sleepDurationProvider: retryAttempt => TimeSpan.FromMilliseconds(
+                        _settings.RetryDelayMs * Math.Pow(2, retryAttempt - 1) + // Exponential backoff
+                        new Random().Next(0, _settings.RetryDelayMs / 4) // Add jitter
+                    ),
+                    onRetry: (outcome, timespan, retryCount, context) =>
+                    {
+                        var exception = outcome.Exception;
+                        var response = outcome.Result;
+                        var errorMessage = exception?.Message ?? $"HTTP {response?.StatusCode}";
+                        
+                        _logger.LogWarning(
+                            "Web3Proxy request failed (attempt {RetryCount}/{MaxRetries}): {Error}. Retrying in {Delay}ms",
+                            retryCount, _settings.MaxRetryAttempts + 1, errorMessage, timespan.TotalMilliseconds);
+                    });
+        }
+
+        private static bool IsRetryableStatusCode(HttpStatusCode statusCode)
+        {
+            return statusCode == HttpStatusCode.RequestTimeout ||
+                   statusCode == HttpStatusCode.TooManyRequests ||
+                   statusCode == HttpStatusCode.InternalServerError ||
+                   statusCode == HttpStatusCode.BadGateway ||
+                   statusCode == HttpStatusCode.ServiceUnavailable ||
+                   statusCode == HttpStatusCode.GatewayTimeout;
+        }
+
+        private async Task<T> ExecuteWithRetryAsync<T>(Func<Task<HttpResponseMessage>> httpCall, string operationName)
+        {
+            try
+            {
+                var response = await _retryPolicy.ExecuteAsync(httpCall);
+                
+                if (!response.IsSuccessStatusCode)
+                {
+                    await HandleErrorResponse(response);
+                }
+
+                var result = await response.Content.ReadFromJsonAsync<T>(_jsonOptions);
+                return result ?? throw new Web3ProxyException($"Failed to deserialize response for {operationName}");
+            }
+            catch (Exception ex) when (!(ex is Web3ProxyException))
+            {
+                _logger.LogError(ex, "Operation {OperationName} failed after all retry attempts", operationName);
+                SentrySdk.CaptureException(ex);
+                throw new Web3ProxyException($"Failed to execute {operationName}: {ex.Message}");
+            }
+        }
+
+        private async Task<T> ExecuteWithRetryAsync<T>(Func<Task<HttpResponseMessage>> httpCall, string operationName, string idempotencyKey)
+        {
+            try
+            {
+                var response = await _retryPolicy.ExecuteAsync(httpCall);
+                
+                if (!response.IsSuccessStatusCode)
+                {
+                    await HandleErrorResponse(response);
+                }
+
+                var result = await response.Content.ReadFromJsonAsync<T>(_jsonOptions);
+                return result ?? throw new Web3ProxyException($"Failed to deserialize response for {operationName}");
+            }
+            catch (Exception ex) when (!(ex is Web3ProxyException))
+            {
+                _logger.LogError(ex, "Operation {OperationName} failed after all retry attempts (IdempotencyKey: {IdempotencyKey})", operationName, idempotencyKey);
+                SentrySdk.CaptureException(ex);
+                throw new Web3ProxyException($"Failed to execute {operationName}: {ex.Message}");
+            }
         }
 
         public async Task<T> CallPrivyServiceAsync<T>(string endpoint, object payload)
@@ -40,26 +129,22 @@ namespace Managing.Infrastructure.Evm.Services
             }
 
             var url = $"{_settings.BaseUrl}/api/privy{endpoint}";
+            var idempotencyKey = Guid.NewGuid().ToString();
 
-            try
-            {
-                var response = await _httpClient.PostAsJsonAsync(url, payload, _jsonOptions);
-
-                if (!response.IsSuccessStatusCode)
-                {
-                    await HandleErrorResponse(response);
-                }
-
-                return await response.Content.ReadFromJsonAsync<T>(_jsonOptions);
-            }
-            catch (Exception ex) when (!(ex is Web3ProxyException))
-            {
-                SentrySdk.CaptureException(ex);
-                throw new Web3ProxyException($"Failed to call Privy service at {endpoint}: {ex.Message}");
-            }
+            return await ExecuteWithRetryAsync<T>(
+                () => {
+                    var request = new HttpRequestMessage(HttpMethod.Post, url)
+                    {
+                        Content = JsonContent.Create(payload, options: _jsonOptions)
+                    };
+                    request.Headers.Add("Idempotency-Key", idempotencyKey);
+                    return _httpClient.SendAsync(request);
+                },
+                $"CallPrivyServiceAsync({endpoint})",
+                idempotencyKey);
         }
 
-        public async Task<T> GetPrivyServiceAsync<T>(string endpoint, object payload = null)
+        public async Task<T> GetPrivyServiceAsync<T>(string endpoint, object? payload = null)
         {
             if (!endpoint.StartsWith("/"))
             {
@@ -73,22 +158,9 @@ namespace Managing.Infrastructure.Evm.Services
                 url += BuildQueryString(payload);
             }
 
-            try
-            {
-                var response = await _httpClient.GetAsync(url);
-
-                if (!response.IsSuccessStatusCode)
-                {
-                    await HandleErrorResponse(response);
-                }
-
-                return await response.Content.ReadFromJsonAsync<T>(_jsonOptions);
-            }
-            catch (Exception ex) when (!(ex is Web3ProxyException))
-            {
-                SentrySdk.CaptureException(ex);
-                throw new Web3ProxyException($"Failed to get Privy service at {endpoint}: {ex.Message}");
-            }
+            return await ExecuteWithRetryAsync<T>(
+                () => _httpClient.GetAsync(url),
+                $"GetPrivyServiceAsync({endpoint})");
         }
 
         public async Task<T> CallGmxServiceAsync<T>(string endpoint, object payload)
@@ -99,26 +171,22 @@ namespace Managing.Infrastructure.Evm.Services
             }
 
             var url = $"{_settings.BaseUrl}/api/gmx{endpoint}";
+            var idempotencyKey = Guid.NewGuid().ToString();
 
-            try
-            {
-                var response = await _httpClient.PostAsJsonAsync(url, payload, _jsonOptions);
-
-                if (!response.IsSuccessStatusCode)
-                {
-                    await HandleErrorResponse(response);
-                }
-
-                return await response.Content.ReadFromJsonAsync<T>(_jsonOptions);
-            }
-            catch (Exception ex) when (!(ex is Web3ProxyException))
-            {
-                SentrySdk.CaptureException(ex);
-                throw new Web3ProxyException($"Failed to call GMX service at {endpoint}: {ex.Message}");
-            }
+            return await ExecuteWithRetryAsync<T>(
+                () => {
+                    var request = new HttpRequestMessage(HttpMethod.Post, url)
+                    {
+                        Content = JsonContent.Create(payload, options: _jsonOptions)
+                    };
+                    request.Headers.Add("Idempotency-Key", idempotencyKey);
+                    return _httpClient.SendAsync(request);
+                },
+                $"CallGmxServiceAsync({endpoint})",
+                idempotencyKey);
         }
 
-        public async Task<T> GetGmxServiceAsync<T>(string endpoint, object payload = null)
+        public async Task<T> GetGmxServiceAsync<T>(string endpoint, object? payload = null)
         {
             if (!endpoint.StartsWith("/"))
             {
@@ -132,22 +200,9 @@ namespace Managing.Infrastructure.Evm.Services
                 url += BuildQueryString(payload);
             }
 
-            try
-            {
-                var response = await _httpClient.GetAsync(url);
-
-                if (!response.IsSuccessStatusCode)
-                {
-                    await HandleErrorResponse(response);
-                }
-
-                return await response.Content.ReadFromJsonAsync<T>(_jsonOptions);
-            }
-            catch (Exception ex) when (!(ex is Web3ProxyException))
-            {
-                SentrySdk.CaptureException(ex);
-                throw new Web3ProxyException($"Failed to get GMX service at {endpoint}: {ex.Message}");
-            }
+            return await ExecuteWithRetryAsync<T>(
+                () => _httpClient.GetAsync(url),
+                $"GetGmxServiceAsync({endpoint})");
         }
 
         public async Task<GmxClaimableSummary> GetGmxClaimableSummaryAsync(string account)