Optimize backtest memory usage by implementing a rolling window for candle storage and update performance benchmarks with new test data.

This commit is contained in:
2025-11-15 13:54:39 +07:00
parent e814eb749c
commit bed25e7222
3 changed files with 15 additions and 4 deletions

View File

@@ -220,6 +220,8 @@ public class BacktestExecutor
// The signal calculation depends on rolling window state and cannot be pre-calculated effectively // The signal calculation depends on rolling window state and cannot be pre-calculated effectively
// Use optimized rolling window approach - TradingBox.GetSignal only needs last 600 candles // Use optimized rolling window approach - TradingBox.GetSignal only needs last 600 candles
const int RollingWindowSize = 600; // TradingBox.GetSignal only needs last 600 candles
var rollingWindowCandles = new Queue<Candle>(RollingWindowSize);
var candlesProcessed = 0; var candlesProcessed = 0;
// Signal caching optimization - reduce signal update frequency for better performance // Signal caching optimization - reduce signal update frequency for better performance
@@ -231,8 +233,6 @@ public class BacktestExecutor
var lastWalletCheck = 0; var lastWalletCheck = 0;
var lastWalletBalance = config.BotTradingBalance; var lastWalletBalance = config.BotTradingBalance;
var fixedCandles = new HashSet<Candle>();
// Track memory usage during processing // Track memory usage during processing
var peakMemory = initialMemory; var peakMemory = initialMemory;
const int memoryCheckInterval = 100; // Check memory every N candles to reduce GC.GetTotalMemory overhead const int memoryCheckInterval = 100; // Check memory every N candles to reduce GC.GetTotalMemory overhead
@@ -252,12 +252,19 @@ public class BacktestExecutor
// Check for cancellation (timeout or shutdown) // Check for cancellation (timeout or shutdown)
cancellationToken.ThrowIfCancellationRequested(); cancellationToken.ThrowIfCancellationRequested();
// Add to HashSet for reuse // Maintain rolling window of last 600 candles to prevent exponential memory growth
fixedCandles.Add(candle); rollingWindowCandles.Enqueue(candle);
if (rollingWindowCandles.Count > RollingWindowSize)
{
rollingWindowCandles.Dequeue(); // Remove oldest candle
}
tradingBot.LastCandle = candle; tradingBot.LastCandle = candle;
// Run with optimized backtest path (minimize async calls) // Run with optimized backtest path (minimize async calls)
var signalUpdateStart = Stopwatch.GetTimestamp(); var signalUpdateStart = Stopwatch.GetTimestamp();
// Convert rolling window to HashSet for TradingBot.UpdateSignals compatibility
var fixedCandles = new HashSet<Candle>(rollingWindowCandles);
await tradingBot.UpdateSignals(fixedCandles, preCalculatedIndicatorValues); await tradingBot.UpdateSignals(fixedCandles, preCalculatedIndicatorValues);
signalUpdateTotalTime += Stopwatch.GetElapsedTime(signalUpdateStart); signalUpdateTotalTime += Stopwatch.GetElapsedTime(signalUpdateStart);

View File

@@ -18,3 +18,5 @@ DateTime,TestName,CandlesCount,ExecutionTimeSeconds,ProcessingRateCandlesPerSec,
2025-11-14T12:43:33Z,Telemetry_ETH_RSI_EMACROSS,5760,6.43,896.2,29.02,22.13,36.21,0.0,0,0.0,0.0,0.0,0.0,-35450.45,20,-49.76,0.00,b60295fc,dev,development 2025-11-14T12:43:33Z,Telemetry_ETH_RSI_EMACROSS,5760,6.43,896.2,29.02,22.13,36.21,0.0,0,0.0,0.0,0.0,0.0,-35450.45,20,-49.76,0.00,b60295fc,dev,development
2025-11-14T12:45:51Z,Telemetry_ETH_RSI_EMACROSS,5760,4.12,1398.2,29.05,21.88,36.24,0.0,0,0.0,0.0,0.0,0.0,-35450.45,20,-49.76,0.00,b60295fc,dev,development 2025-11-14T12:45:51Z,Telemetry_ETH_RSI_EMACROSS,5760,4.12,1398.2,29.05,21.88,36.24,0.0,0,0.0,0.0,0.0,0.0,-35450.45,20,-49.76,0.00,b60295fc,dev,development
2025-11-14T12:46:43Z,Telemetry_ETH_RSI_EMACROSS,5760,3.86,1491.9,29.05,20.88,36.27,0.0,0,0.0,0.0,0.0,0.0,-35450.45,20,-49.76,0.00,b60295fc,dev,development 2025-11-14T12:46:43Z,Telemetry_ETH_RSI_EMACROSS,5760,3.86,1491.9,29.05,20.88,36.27,0.0,0,0.0,0.0,0.0,0.0,-35450.45,20,-49.76,0.00,b60295fc,dev,development
2025-11-15T06:46:21Z,Telemetry_ETH_RSI_EMACROSS,5760,12.58,457.8,28.82,21.79,35.28,0.0,0,0.0,0.0,0.0,0.0,-35450.45,20,-49.76,0.00,e814eb74,dev,development
2025-11-15T06:50:04Z,Telemetry_ETH_RSI_EMACROSS,5760,4.84,1190.4,29.01,19.10,35.17,0.0,0,0.0,0.0,0.0,0.0,-35450.45,20,-49.76,0.00,e814eb74,dev,development
1 DateTime TestName CandlesCount ExecutionTimeSeconds ProcessingRateCandlesPerSec MemoryStartMB MemoryEndMB MemoryPeakMB SignalUpdatesCount SignalUpdatesSkipped SignalUpdateEfficiencyPercent BacktestStepsCount AverageSignalUpdateMs AverageBacktestStepMs FinalPnL WinRatePercent GrowthPercentage Score CommitHash GitBranch Environment
18 2025-11-14T12:43:33Z Telemetry_ETH_RSI_EMACROSS 5760 6.43 896.2 29.02 22.13 36.21 0.0 0 0.0 0.0 0.0 0.0 -35450.45 20 -49.76 0.00 b60295fc dev development
19 2025-11-14T12:45:51Z Telemetry_ETH_RSI_EMACROSS 5760 4.12 1398.2 29.05 21.88 36.24 0.0 0 0.0 0.0 0.0 0.0 -35450.45 20 -49.76 0.00 b60295fc dev development
20 2025-11-14T12:46:43Z Telemetry_ETH_RSI_EMACROSS 5760 3.86 1491.9 29.05 20.88 36.27 0.0 0 0.0 0.0 0.0 0.0 -35450.45 20 -49.76 0.00 b60295fc dev development
21 2025-11-15T06:46:21Z Telemetry_ETH_RSI_EMACROSS 5760 12.58 457.8 28.82 21.79 35.28 0.0 0 0.0 0.0 0.0 0.0 -35450.45 20 -49.76 0.00 e814eb74 dev development
22 2025-11-15T06:50:04Z Telemetry_ETH_RSI_EMACROSS 5760 4.84 1190.4 29.01 19.10 35.17 0.0 0 0.0 0.0 0.0 0.0 -35450.45 20 -49.76 0.00 e814eb74 dev development

View File

@@ -63,3 +63,5 @@ DateTime,TestName,CandlesCount,ExecutionTimeSeconds,ProcessingRateCandlesPerSec,
2025-11-14T12:43:33Z,Telemetry_ETH_RSI,5760,4.345,1324.3,21.03,20.35,28.95,0.00,0,0.0,4210.13,0.00,0.73,-30689.97,24,-51.70,0.00,b60295fc,dev,development 2025-11-14T12:43:33Z,Telemetry_ETH_RSI,5760,4.345,1324.3,21.03,20.35,28.95,0.00,0,0.0,4210.13,0.00,0.73,-30689.97,24,-51.70,0.00,b60295fc,dev,development
2025-11-14T12:45:51Z,Telemetry_ETH_RSI,5760,2.235,2576.6,29.03,20.37,36.11,0.00,0,0.0,2103.00,0.00,0.37,-30689.97,24,-51.70,0.00,b60295fc,dev,development 2025-11-14T12:45:51Z,Telemetry_ETH_RSI,5760,2.235,2576.6,29.03,20.37,36.11,0.00,0,0.0,2103.00,0.00,0.37,-30689.97,24,-51.70,0.00,b60295fc,dev,development
2025-11-14T12:46:43Z,Telemetry_ETH_RSI,5760,3.44,1669.8,28.85,20.34,35.90,0.00,0,0.0,3304.34,0.00,0.57,-30689.97,24,-51.70,0.00,b60295fc,dev,development 2025-11-14T12:46:43Z,Telemetry_ETH_RSI,5760,3.44,1669.8,28.85,20.34,35.90,0.00,0,0.0,3304.34,0.00,0.57,-30689.97,24,-51.70,0.00,b60295fc,dev,development
2025-11-15T06:46:21Z,Telemetry_ETH_RSI,5760,4.83,1191.0,29.02,20.22,37.20,4105.51,0,0.0,499.39,0.00,0.09,-30689.97,24,-51.70,0.00,e814eb74,dev,development
2025-11-15T06:50:04Z,Telemetry_ETH_RSI,5760,4.47,1286.2,28.81,20.58,34.89,3324.75,0,0.0,965.71,0.00,0.17,-30689.97,24,-51.70,0.00,e814eb74,dev,development
1 DateTime TestName CandlesCount ExecutionTimeSeconds ProcessingRateCandlesPerSec MemoryStartMB MemoryEndMB MemoryPeakMB SignalUpdatesCount SignalUpdatesSkipped SignalUpdateEfficiencyPercent BacktestStepsCount AverageSignalUpdateMs AverageBacktestStepMs FinalPnL WinRatePercent GrowthPercentage Score CommitHash GitBranch Environment
63 2025-11-14T12:43:33Z Telemetry_ETH_RSI 5760 4.345 1324.3 21.03 20.35 28.95 0.00 0 0.0 4210.13 0.00 0.73 -30689.97 24 -51.70 0.00 b60295fc dev development
64 2025-11-14T12:45:51Z Telemetry_ETH_RSI 5760 2.235 2576.6 29.03 20.37 36.11 0.00 0 0.0 2103.00 0.00 0.37 -30689.97 24 -51.70 0.00 b60295fc dev development
65 2025-11-14T12:46:43Z Telemetry_ETH_RSI 5760 3.44 1669.8 28.85 20.34 35.90 0.00 0 0.0 3304.34 0.00 0.57 -30689.97 24 -51.70 0.00 b60295fc dev development
66 2025-11-15T06:46:21Z Telemetry_ETH_RSI 5760 4.83 1191.0 29.02 20.22 37.20 4105.51 0 0.0 499.39 0.00 0.09 -30689.97 24 -51.70 0.00 e814eb74 dev development
67 2025-11-15T06:50:04Z Telemetry_ETH_RSI 5760 4.47 1286.2 28.81 20.58 34.89 3324.75 0 0.0 965.71 0.00 0.17 -30689.97 24 -51.70 0.00 e814eb74 dev development