diff --git a/src/Managing.Application/Backtests/BacktestExecutor.cs b/src/Managing.Application/Backtests/BacktestExecutor.cs index 7e07e20d..b835c295 100644 --- a/src/Managing.Application/Backtests/BacktestExecutor.cs +++ b/src/Managing.Application/Backtests/BacktestExecutor.cs @@ -177,9 +177,10 @@ public class BacktestExecutor // Pre-calculate indicator values once for all candles to optimize performance // This avoids recalculating indicators for every candle iteration Dictionary preCalculatedIndicatorValues = null; - var indicatorCalcStart = Stopwatch.GetTimestamp(); - if (config.Scenario != null) + if (config.Scenario != null && false) { + var indicatorCalcStart = Stopwatch.GetTimestamp(); + try { _logger.LogInformation("⚡ Pre-calculating indicator values for {IndicatorCount} indicators", @@ -222,15 +223,12 @@ public class BacktestExecutor var initialBalance = config.BotTradingBalance; // Pre-allocate and populate candle structures for maximum performance - var orderedCandles = candles.OrderBy(c => c.Date).ToList(); + var orderedCandles = candles.ToList(); // Skip pre-calculated signals - the approach was flawed and caused performance regression // The signal calculation depends on rolling window state and cannot be pre-calculated effectively // Use optimized rolling window approach - TradingBox.GetSignal only needs last 600 candles - const int rollingWindowSize = 600; - var rollingCandles = new List(rollingWindowSize); // Pre-allocate capacity for better performance - var fixedCandlesHashSet = new HashSet(rollingWindowSize); // Reuse HashSet to avoid allocations var candlesProcessed = 0; // Signal caching optimization - reduce signal update frequency for better performance @@ -242,6 +240,8 @@ public class BacktestExecutor var lastWalletCheck = 0; var lastWalletBalance = config.BotTradingBalance; + var fixedCandles = new HashSet(); + // Track memory usage during processing var peakMemory = initialMemory; const int memoryCheckInterval = 100; // Check memory every N candles to reduce GC.GetTotalMemory overhead @@ -258,44 +258,15 @@ public class BacktestExecutor // Process all candles with optimized rolling window approach foreach (var candle in orderedCandles) { - // Maintain rolling window efficiently using List - rollingCandles.Add(candle); - - if (rollingCandles.Count > rollingWindowSize) - { - // Remove oldest candle from both structures - var removedCandle = rollingCandles[0]; - rollingCandles.RemoveAt(0); - fixedCandlesHashSet.Remove(removedCandle); - } - // Add to HashSet for reuse - fixedCandlesHashSet.Add(candle); + fixedCandles.Add(candle); tradingBot.LastCandle = candle; - // Smart signal caching - reduce signal update frequency for performance - // RSI and similar indicators don't need updates every candle for 15-minute data - var shouldSkipSignalUpdate = ShouldSkipSignalUpdate(currentCandle, totalCandles); - - if (!shouldSkipSignalUpdate) - { - // Smart signal caching - reduce signal update frequency for performance - // RSI and similar indicators don't need updates every candle for 15-minute data - var signalUpdateStart = Stopwatch.GetTimestamp(); - await tradingBot.UpdateSignals(fixedCandlesHashSet); - signalUpdateTotalTime += Stopwatch.GetElapsedTime(signalUpdateStart); - telemetry.TotalSignalUpdates++; - } - else - { - signalUpdateSkipCount++; - // Skip signal update - reuse previous signal state - // This saves ~1ms per skipped update and improves performance significantly - } - // Run with optimized backtest path (minimize async calls) var backtestStepStart = Stopwatch.GetTimestamp(); - await RunOptimizedBacktestStep(tradingBot); + await tradingBot.UpdateSignals(fixedCandles); + await tradingBot.Run(); + backtestStepTotalTime += Stopwatch.GetElapsedTime(backtestStepStart); telemetry.TotalBacktestSteps++; @@ -343,11 +314,12 @@ public class BacktestExecutor // Track peak memory usage (reduced frequency to minimize GC overhead) if (currentCandle - lastMemoryCheck >= memoryCheckInterval) { - var currentMemory = GC.GetTotalMemory(false); - if (currentMemory > peakMemory) - { - peakMemory = currentMemory; + var currentMemory = GC.GetTotalMemory(false); + if (currentMemory > peakMemory) + { + peakMemory = currentMemory; } + lastMemoryCheck = currentCandle; } @@ -554,7 +526,6 @@ public class BacktestExecutor /// Pre-calculates all signals for the entire backtest period /// This eliminates repeated GetSignal() calls during the backtest loop /// - /// /// Converts a Backtest to LightBacktest /// @@ -609,7 +580,6 @@ public class BacktestExecutor private async Task RunOptimizedBacktestStep(TradingBotBase tradingBot) { // Use the standard Run method but ensure it's optimized for backtests - await tradingBot.Run(); } diff --git a/src/Managing.Application/Bots/TradingBotBase.cs b/src/Managing.Application/Bots/TradingBotBase.cs index 3502631d..e103bc27 100644 --- a/src/Managing.Application/Bots/TradingBotBase.cs +++ b/src/Managing.Application/Bots/TradingBotBase.cs @@ -263,7 +263,8 @@ public class TradingBotBase : ITradingBot await UpdateSignals(candles, null); } - public async Task UpdateSignals(HashSet candles, Dictionary preCalculatedSignals = null) + public async Task UpdateSignals(HashSet candles, + Dictionary preCalculatedSignals = null) { // Skip indicator checking if flipping is disabled and there's an open position // This prevents unnecessary indicator calculations when we can't act on signals anyway @@ -283,27 +284,10 @@ public class TradingBotBase : ITradingBot if (Config.IsForBacktest) { - LightSignal backtestSignal; - - if (preCalculatedSignals != null && LastCandle != null && preCalculatedSignals.TryGetValue(LastCandle.Date, out backtestSignal)) - { - // Use pre-calculated signal - fast path - if (backtestSignal == null) return; - await AddSignal(backtestSignal); - } - else if (candles != null) - { - // Fallback to original calculation if no pre-calculated signals available - backtestSignal = TradingBox.GetSignal(candles, Config.Scenario, Signals, Config.Scenario.LoopbackPeriod, - PreCalculatedIndicatorValues); - if (backtestSignal == null) return; - await AddSignal(backtestSignal); - } - else - { - // No candles provided - skip signal update - return; - } + var backtestSignal = TradingBox.GetSignal(candles, Config.Scenario, Signals, Config.Scenario.LoopbackPeriod, + PreCalculatedIndicatorValues); + if (backtestSignal == null) return; + await AddSignal(backtestSignal); } else { @@ -601,7 +585,8 @@ public class TradingBotBase : ITradingBot $"Checking position history before marking as closed..."); // Verify in exchange history before assuming it's closed - var (existsInHistory, hadWeb3ProxyError) = await CheckPositionInExchangeHistory(positionForSignal); + var (existsInHistory, hadWeb3ProxyError) = + await CheckPositionInExchangeHistory(positionForSignal); if (hadWeb3ProxyError) { @@ -781,7 +766,8 @@ public class TradingBotBase : ITradingBot // Position might be canceled by the broker // Check if position exists in exchange history with PnL before canceling - var (positionFoundInHistory, hadWeb3ProxyError) = await CheckPositionInExchangeHistory(positionForSignal); + var (positionFoundInHistory, hadWeb3ProxyError) = + await CheckPositionInExchangeHistory(positionForSignal); if (hadWeb3ProxyError) { @@ -3087,7 +3073,8 @@ public class TradingBotBase : ITradingBot } catch (Exception ex) { - Logger.LogError(ex, "Error during position recovery for position {PositionId}", positionForSignal.Identifier); + Logger.LogError(ex, "Error during position recovery for position {PositionId}", + positionForSignal.Identifier); await LogWarning($"Position recovery failed due to exception: {ex.Message}"); return false; } diff --git a/src/Managing.Domain/Indicators/Signals/EmaCrossIndicator.cs b/src/Managing.Domain/Indicators/Signals/EmaCrossIndicator.cs index 25b1b6e2..31e48cc2 100644 --- a/src/Managing.Domain/Indicators/Signals/EmaCrossIndicator.cs +++ b/src/Managing.Domain/Indicators/Signals/EmaCrossIndicator.cs @@ -68,7 +68,6 @@ public class EmaCrossIndicator : EmaBaseIndicatorBase // Filter pre-calculated EMA values to match the candles we're processing ema = preCalculatedValues.Ema .Where(e => candles.Any(c => c.Date == e.Date)) - .OrderBy(e => e.Date) .ToList(); } diff --git a/src/Managing.Domain/Shared/Helpers/TradingBox.cs b/src/Managing.Domain/Shared/Helpers/TradingBox.cs index 3b0fe887..ec2cfcc8 100644 --- a/src/Managing.Domain/Shared/Helpers/TradingBox.cs +++ b/src/Managing.Domain/Shared/Helpers/TradingBox.cs @@ -61,7 +61,8 @@ public static class TradingBox Dictionary previousSignal, int? loopbackPeriod, Dictionary preCalculatedIndicatorValues) { - return GetSignal(newCandles, scenario, previousSignal, _defaultConfig, loopbackPeriod, preCalculatedIndicatorValues); + return GetSignal(newCandles, scenario, previousSignal, _defaultConfig, loopbackPeriod, + preCalculatedIndicatorValues); } public static LightSignal GetSignal(HashSet newCandles, LightScenario lightScenario, @@ -75,24 +76,11 @@ public static class TradingBox Dictionary preCalculatedIndicatorValues) { var signalOnCandles = new List(); - // Optimize list creation - avoid redundant allocations and multiple ordering - List limitedCandles; - if (newCandles.Count <= 600) - { - // For small sets, just order once - limitedCandles = newCandles.OrderBy(c => c.Date).ToList(); - } - else - { - // For large sets, use more efficient approach: sort then take last - var sorted = newCandles.OrderBy(c => c.Date).ToList(); - limitedCandles = sorted.Skip(sorted.Count - 600).ToList(); - } foreach (var indicator in lightScenario.Indicators) { IIndicator indicatorInstance = indicator.ToInterface(); - + // Use pre-calculated indicator values if available (for backtest optimization) List signals; if (preCalculatedIndicatorValues != null && preCalculatedIndicatorValues.ContainsKey(indicator.Type)) @@ -122,10 +110,11 @@ public static class TradingBox continue; } + var limitedCandles = newCandles.ToList(); // Optimized: limitedCandles is already ordered, no need to re-order var loopback = loopbackPeriod.HasValue && loopbackPeriod > 1 ? loopbackPeriod.Value : 1; - var candleLoopback = limitedCandles.Count > loopback - ? limitedCandles.Skip(limitedCandles.Count - loopback).ToList() + var candleLoopback = limitedCandles.Count > loopback + ? limitedCandles.Skip(limitedCandles.Count - loopback).ToList() : limitedCandles; if (!candleLoopback.Any())