Add yield for orleans + 1min to 2h timeout for grain message + more exception send to sentry

src/Managing.Application/Bots/Grains/BacktestTradingBotGrain.cs

@@ -82,6 +82,10 @@ public class BacktestTradingBotGrain : Grain, IBacktestTradingBotGrain
         var initialBalance = config.BotTradingBalance;
 
         var fixedCandles = new HashSet<Candle>();
+        var lastYieldTime = DateTime.UtcNow;
+        const int yieldIntervalMs = 5000; // Yield control every 5 seconds to prevent timeout
+        const int candlesPerBatch = 100; // Process in batches to allow Orleans to check for cancellation
+        
         // Process all candles following the exact pattern from GetBacktestingResult
         foreach (var candle in candles)
         {
@@ -94,6 +98,16 @@ public class BacktestTradingBotGrain : Grain, IBacktestTradingBotGrain
 
             currentCandle++;
 
+            // Yield control periodically to prevent Orleans from thinking the grain is stuck
+            // This helps prevent timeout issues during long-running backtests
+            var timeSinceLastYield = (DateTime.UtcNow - lastYieldTime).TotalMilliseconds;
+            if (timeSinceLastYield >= yieldIntervalMs || currentCandle % candlesPerBatch == 0)
+            {
+                // Yield control back to Orleans scheduler
+                await Task.Yield();
+                lastYieldTime = DateTime.UtcNow;
+            }
+
             // Log progress every 10%
             var currentPercentage = (currentCandle * 100) / totalCandles;
             if (currentPercentage >= lastLoggedPercentage + 10)

src/Managing.Application/Grains/BundleBacktestGrain.cs

@@ -126,6 +126,7 @@ public class BundleBacktestGrain : Grain, IBundleBacktestGrain, IRemindable
         catch (Exception ex)
         {
             _logger.LogError(ex, "Error processing bundle backtest request {RequestId}", bundleRequest.RequestId);
+            SentrySdk.CaptureException(ex);
             await HandleBundleRequestError(bundleRequest, backtester, ex);
         }
     }
@@ -133,34 +134,41 @@ public class BundleBacktestGrain : Grain, IBundleBacktestGrain, IRemindable
     /// <summary>
     /// Generates individual backtest requests from variant configuration
     /// </summary>
-    private async Task<List<RunBacktestRequest>> GenerateBacktestRequestsFromVariants(BundleBacktestRequest bundleRequest)
+    private async Task<List<RunBacktestRequest>> GenerateBacktestRequestsFromVariants(
+        BundleBacktestRequest bundleRequest)
     {
         try
         {
             // Deserialize the variant configurations
-            var universalConfig = JsonSerializer.Deserialize<BundleBacktestUniversalConfig>(bundleRequest.UniversalConfigJson);
+            var universalConfig =
+                JsonSerializer.Deserialize<BundleBacktestUniversalConfig>(bundleRequest.UniversalConfigJson);
             var dateTimeRanges = JsonSerializer.Deserialize<List<DateTimeRange>>(bundleRequest.DateTimeRangesJson);
-            var moneyManagementVariants = JsonSerializer.Deserialize<List<MoneyManagementVariant>>(bundleRequest.MoneyManagementVariantsJson);
+            var moneyManagementVariants =
+                JsonSerializer.Deserialize<List<MoneyManagementVariant>>(bundleRequest.MoneyManagementVariantsJson);
             var tickerVariants = JsonSerializer.Deserialize<List<Ticker>>(bundleRequest.TickerVariantsJson);
 
-            if (universalConfig == null || dateTimeRanges == null || moneyManagementVariants == null || tickerVariants == null)
+            if (universalConfig == null || dateTimeRanges == null || moneyManagementVariants == null ||
+                tickerVariants == null)
             {
-                _logger.LogError("Failed to deserialize variant configurations for bundle request {RequestId}", bundleRequest.RequestId);
+                _logger.LogError("Failed to deserialize variant configurations for bundle request {RequestId}",
+                    bundleRequest.RequestId);
                 return new List<RunBacktestRequest>();
             }
 
             // Get the first account for the user using AccountService
             var firstAccount = await ServiceScopeHelpers.WithScopedService<IAccountService, Account?>(
                 _scopeFactory,
-                async service => 
+                async service =>
                 {
-                    var accounts = await service.GetAccountsByUserAsync(bundleRequest.User, hideSecrets: true, getBalance: false);
+                    var accounts =
+                        await service.GetAccountsByUserAsync(bundleRequest.User, hideSecrets: true, getBalance: false);
                     return accounts.FirstOrDefault();
                 });
 
             if (firstAccount == null)
             {
-                _logger.LogError("No accounts found for user {UserId} in bundle request {RequestId}", bundleRequest.User.Id, bundleRequest.RequestId);
+                _logger.LogError("No accounts found for user {UserId} in bundle request {RequestId}",
+                    bundleRequest.User.Id, bundleRequest.RequestId);
                 return new List<RunBacktestRequest>();
             }
 
@@ -179,7 +187,8 @@ public class BundleBacktestGrain : Grain, IBundleBacktestGrain, IRemindable
                             Timeframe = universalConfig.Timeframe,
                             IsForWatchingOnly = universalConfig.IsForWatchingOnly,
                             BotTradingBalance = universalConfig.BotTradingBalance,
-                            Name = $"{universalConfig.BotName}_{ticker}_{dateRange.StartDate:yyyyMMdd}_{dateRange.EndDate:yyyyMMdd}",
+                            Name =
+                                $"{universalConfig.BotName}_{ticker}_{dateRange.StartDate:yyyyMMdd}_{dateRange.EndDate:yyyyMMdd}",
                             FlipPosition = universalConfig.FlipPosition,
                             CooldownPeriod = universalConfig.CooldownPeriod,
                             MaxLossStreak = universalConfig.MaxLossStreak,
@@ -204,15 +213,16 @@ public class BundleBacktestGrain : Grain, IBundleBacktestGrain, IRemindable
                             WatchOnly = universalConfig.WatchOnly,
                             Save = universalConfig.Save,
                             WithCandles = false, // Bundle backtests never return candles
-                            MoneyManagement = mmVariant.MoneyManagement != null ? 
+                            MoneyManagement = mmVariant.MoneyManagement != null
-                                new MoneyManagement
+                                ? new MoneyManagement
                                 {
                                     Name = mmVariant.MoneyManagement.Name,
                                     Timeframe = mmVariant.MoneyManagement.Timeframe,
                                     StopLoss = mmVariant.MoneyManagement.StopLoss,
                                     TakeProfit = mmVariant.MoneyManagement.TakeProfit,
                                     Leverage = mmVariant.MoneyManagement.Leverage
-                                } : null
+                                }
+                                : null
                         };
 
                         backtestRequests.Add(backtestRequest);
@@ -224,7 +234,8 @@ public class BundleBacktestGrain : Grain, IBundleBacktestGrain, IRemindable
         }
         catch (Exception ex)
         {
-            _logger.LogError(ex, "Error generating backtest requests from variants for bundle request {RequestId}", bundleRequest.RequestId);
+            _logger.LogError(ex, "Error generating backtest requests from variants for bundle request {RequestId}",
+                bundleRequest.RequestId);
             return new List<RunBacktestRequest>();
         }
     }
@@ -267,6 +278,7 @@ public class BundleBacktestGrain : Grain, IBundleBacktestGrain, IRemindable
                 index + 1, bundleRequest.RequestId);
             bundleRequest.FailedBacktests++;
             await backtester.UpdateBundleBacktestRequestAsync(bundleRequest);
+            SentrySdk.CaptureException(ex);
         }
     }
 

src/Managing.Bootstrap/ApiBootstrap.cs

@@ -238,7 +238,8 @@ public static class ApiBootstrap
                     .Configure<MessagingOptions>(options =>
                     {
                         // Configure messaging for better reliability with increased timeouts
-                        options.ResponseTimeout = TimeSpan.FromSeconds(60);
+                        // Set to 2 hours to support long-running backtests that can take 47+ minutes
+                        options.ResponseTimeout = TimeSpan.FromHours(2);
                         options.DropExpiredMessages = true;
                     })
                     .Configure<ClusterMembershipOptions>(options =>
@@ -265,7 +266,9 @@ public static class ApiBootstrap
                     siloBuilder.Configure<GrainCollectionOptions>(options =>
                     {
                         // Enable grain collection for active grains
-                        options.CollectionAge = TimeSpan.FromMinutes(10);
+                        // Set to 2.5 hours to allow long-running backtests (up to 2 hours) to complete
+                        // without being collected prematurely
+                        options.CollectionAge = TimeSpan.FromMinutes(150); // 2.5 hours
                         options.CollectionQuantum = TimeSpan.FromMinutes(1);
                     });
                 }
@@ -281,7 +284,16 @@ public static class ApiBootstrap
                 }
 
                 siloBuilder
-                    .ConfigureLogging(logging => logging.SetMinimumLevel(LogLevel.Information));
+                    .ConfigureLogging(logging =>
+                    {
+                        logging.SetMinimumLevel(LogLevel.Information);
+                        // Reduce verbosity of Orleans status update messages for long-running operations
+                        // These are informational and can be very verbose during long backtests
+                        logging.AddFilter("Orleans.Runtime.Messaging.IncomingMessageAcceptor", LogLevel.Warning);
+                        logging.AddFilter("Orleans.Runtime.Messaging.MessageCenter", LogLevel.Warning);
+                        // Keep important Orleans logs but reduce status update noise
+                        logging.AddFilter("Microsoft.Orleans.Runtime.Messaging", LogLevel.Warning);
+                    });
 
                 // Only enable dashboard in development to avoid shutdown issues
                 if (!isProduction)