diff --git a/src/Managing.Application/Bots/Grains/BacktestTradingBotGrain.cs b/src/Managing.Application/Bots/Grains/BacktestTradingBotGrain.cs index 1f1a2490..56bbdfef 100644 --- a/src/Managing.Application/Bots/Grains/BacktestTradingBotGrain.cs +++ b/src/Managing.Application/Bots/Grains/BacktestTradingBotGrain.cs @@ -82,6 +82,10 @@ public class BacktestTradingBotGrain : Grain, IBacktestTradingBotGrain var initialBalance = config.BotTradingBalance; var fixedCandles = new HashSet(); + var lastYieldTime = DateTime.UtcNow; + const int yieldIntervalMs = 5000; // Yield control every 5 seconds to prevent timeout + const int candlesPerBatch = 100; // Process in batches to allow Orleans to check for cancellation + // Process all candles following the exact pattern from GetBacktestingResult foreach (var candle in candles) { @@ -94,6 +98,16 @@ public class BacktestTradingBotGrain : Grain, IBacktestTradingBotGrain currentCandle++; + // Yield control periodically to prevent Orleans from thinking the grain is stuck + // This helps prevent timeout issues during long-running backtests + var timeSinceLastYield = (DateTime.UtcNow - lastYieldTime).TotalMilliseconds; + if (timeSinceLastYield >= yieldIntervalMs || currentCandle % candlesPerBatch == 0) + { + // Yield control back to Orleans scheduler + await Task.Yield(); + lastYieldTime = DateTime.UtcNow; + } + // Log progress every 10% var currentPercentage = (currentCandle * 100) / totalCandles; if (currentPercentage >= lastLoggedPercentage + 10) diff --git a/src/Managing.Application/Grains/BundleBacktestGrain.cs b/src/Managing.Application/Grains/BundleBacktestGrain.cs index 20021379..882b2cfe 100644 --- a/src/Managing.Application/Grains/BundleBacktestGrain.cs +++ b/src/Managing.Application/Grains/BundleBacktestGrain.cs @@ -126,6 +126,7 @@ public class BundleBacktestGrain : Grain, IBundleBacktestGrain, IRemindable catch (Exception ex) { _logger.LogError(ex, "Error processing bundle backtest request {RequestId}", bundleRequest.RequestId); + SentrySdk.CaptureException(ex); await HandleBundleRequestError(bundleRequest, backtester, ex); } } @@ -133,34 +134,41 @@ public class BundleBacktestGrain : Grain, IBundleBacktestGrain, IRemindable /// /// Generates individual backtest requests from variant configuration /// - private async Task> GenerateBacktestRequestsFromVariants(BundleBacktestRequest bundleRequest) + private async Task> GenerateBacktestRequestsFromVariants( + BundleBacktestRequest bundleRequest) { try { // Deserialize the variant configurations - var universalConfig = JsonSerializer.Deserialize(bundleRequest.UniversalConfigJson); + var universalConfig = + JsonSerializer.Deserialize(bundleRequest.UniversalConfigJson); var dateTimeRanges = JsonSerializer.Deserialize>(bundleRequest.DateTimeRangesJson); - var moneyManagementVariants = JsonSerializer.Deserialize>(bundleRequest.MoneyManagementVariantsJson); + var moneyManagementVariants = + JsonSerializer.Deserialize>(bundleRequest.MoneyManagementVariantsJson); var tickerVariants = JsonSerializer.Deserialize>(bundleRequest.TickerVariantsJson); - if (universalConfig == null || dateTimeRanges == null || moneyManagementVariants == null || tickerVariants == null) + if (universalConfig == null || dateTimeRanges == null || moneyManagementVariants == null || + tickerVariants == null) { - _logger.LogError("Failed to deserialize variant configurations for bundle request {RequestId}", bundleRequest.RequestId); + _logger.LogError("Failed to deserialize variant configurations for bundle request {RequestId}", + bundleRequest.RequestId); return new List(); } // Get the first account for the user using AccountService var firstAccount = await ServiceScopeHelpers.WithScopedService( _scopeFactory, - async service => + async service => { - var accounts = await service.GetAccountsByUserAsync(bundleRequest.User, hideSecrets: true, getBalance: false); + var accounts = + await service.GetAccountsByUserAsync(bundleRequest.User, hideSecrets: true, getBalance: false); return accounts.FirstOrDefault(); }); if (firstAccount == null) { - _logger.LogError("No accounts found for user {UserId} in bundle request {RequestId}", bundleRequest.User.Id, bundleRequest.RequestId); + _logger.LogError("No accounts found for user {UserId} in bundle request {RequestId}", + bundleRequest.User.Id, bundleRequest.RequestId); return new List(); } @@ -179,7 +187,8 @@ public class BundleBacktestGrain : Grain, IBundleBacktestGrain, IRemindable Timeframe = universalConfig.Timeframe, IsForWatchingOnly = universalConfig.IsForWatchingOnly, BotTradingBalance = universalConfig.BotTradingBalance, - Name = $"{universalConfig.BotName}_{ticker}_{dateRange.StartDate:yyyyMMdd}_{dateRange.EndDate:yyyyMMdd}", + Name = + $"{universalConfig.BotName}_{ticker}_{dateRange.StartDate:yyyyMMdd}_{dateRange.EndDate:yyyyMMdd}", FlipPosition = universalConfig.FlipPosition, CooldownPeriod = universalConfig.CooldownPeriod, MaxLossStreak = universalConfig.MaxLossStreak, @@ -204,15 +213,16 @@ public class BundleBacktestGrain : Grain, IBundleBacktestGrain, IRemindable WatchOnly = universalConfig.WatchOnly, Save = universalConfig.Save, WithCandles = false, // Bundle backtests never return candles - MoneyManagement = mmVariant.MoneyManagement != null ? - new MoneyManagement + MoneyManagement = mmVariant.MoneyManagement != null + ? new MoneyManagement { Name = mmVariant.MoneyManagement.Name, Timeframe = mmVariant.MoneyManagement.Timeframe, StopLoss = mmVariant.MoneyManagement.StopLoss, TakeProfit = mmVariant.MoneyManagement.TakeProfit, Leverage = mmVariant.MoneyManagement.Leverage - } : null + } + : null }; backtestRequests.Add(backtestRequest); @@ -224,7 +234,8 @@ public class BundleBacktestGrain : Grain, IBundleBacktestGrain, IRemindable } catch (Exception ex) { - _logger.LogError(ex, "Error generating backtest requests from variants for bundle request {RequestId}", bundleRequest.RequestId); + _logger.LogError(ex, "Error generating backtest requests from variants for bundle request {RequestId}", + bundleRequest.RequestId); return new List(); } } @@ -267,6 +278,7 @@ public class BundleBacktestGrain : Grain, IBundleBacktestGrain, IRemindable index + 1, bundleRequest.RequestId); bundleRequest.FailedBacktests++; await backtester.UpdateBundleBacktestRequestAsync(bundleRequest); + SentrySdk.CaptureException(ex); } } diff --git a/src/Managing.Bootstrap/ApiBootstrap.cs b/src/Managing.Bootstrap/ApiBootstrap.cs index d370f2ec..c99ad7a1 100644 --- a/src/Managing.Bootstrap/ApiBootstrap.cs +++ b/src/Managing.Bootstrap/ApiBootstrap.cs @@ -238,7 +238,8 @@ public static class ApiBootstrap .Configure(options => { // Configure messaging for better reliability with increased timeouts - options.ResponseTimeout = TimeSpan.FromSeconds(60); + // Set to 2 hours to support long-running backtests that can take 47+ minutes + options.ResponseTimeout = TimeSpan.FromHours(2); options.DropExpiredMessages = true; }) .Configure(options => @@ -265,7 +266,9 @@ public static class ApiBootstrap siloBuilder.Configure(options => { // Enable grain collection for active grains - options.CollectionAge = TimeSpan.FromMinutes(10); + // Set to 2.5 hours to allow long-running backtests (up to 2 hours) to complete + // without being collected prematurely + options.CollectionAge = TimeSpan.FromMinutes(150); // 2.5 hours options.CollectionQuantum = TimeSpan.FromMinutes(1); }); } @@ -281,7 +284,16 @@ public static class ApiBootstrap } siloBuilder - .ConfigureLogging(logging => logging.SetMinimumLevel(LogLevel.Information)); + .ConfigureLogging(logging => + { + logging.SetMinimumLevel(LogLevel.Information); + // Reduce verbosity of Orleans status update messages for long-running operations + // These are informational and can be very verbose during long backtests + logging.AddFilter("Orleans.Runtime.Messaging.IncomingMessageAcceptor", LogLevel.Warning); + logging.AddFilter("Orleans.Runtime.Messaging.MessageCenter", LogLevel.Warning); + // Keep important Orleans logs but reduce status update noise + logging.AddFilter("Microsoft.Orleans.Runtime.Messaging", LogLevel.Warning); + }); // Only enable dashboard in development to avoid shutdown issues if (!isProduction)