Optimize backtest memory usage by implementing a rolling window for candle storage and update performance benchmarks with new test data.
This commit is contained in:
@@ -220,6 +220,8 @@ public class BacktestExecutor
|
|||||||
// The signal calculation depends on rolling window state and cannot be pre-calculated effectively
|
// The signal calculation depends on rolling window state and cannot be pre-calculated effectively
|
||||||
|
|
||||||
// Use optimized rolling window approach - TradingBox.GetSignal only needs last 600 candles
|
// Use optimized rolling window approach - TradingBox.GetSignal only needs last 600 candles
|
||||||
|
const int RollingWindowSize = 600; // TradingBox.GetSignal only needs last 600 candles
|
||||||
|
var rollingWindowCandles = new Queue<Candle>(RollingWindowSize);
|
||||||
var candlesProcessed = 0;
|
var candlesProcessed = 0;
|
||||||
|
|
||||||
// Signal caching optimization - reduce signal update frequency for better performance
|
// Signal caching optimization - reduce signal update frequency for better performance
|
||||||
@@ -231,8 +233,6 @@ public class BacktestExecutor
|
|||||||
var lastWalletCheck = 0;
|
var lastWalletCheck = 0;
|
||||||
var lastWalletBalance = config.BotTradingBalance;
|
var lastWalletBalance = config.BotTradingBalance;
|
||||||
|
|
||||||
var fixedCandles = new HashSet<Candle>();
|
|
||||||
|
|
||||||
// Track memory usage during processing
|
// Track memory usage during processing
|
||||||
var peakMemory = initialMemory;
|
var peakMemory = initialMemory;
|
||||||
const int memoryCheckInterval = 100; // Check memory every N candles to reduce GC.GetTotalMemory overhead
|
const int memoryCheckInterval = 100; // Check memory every N candles to reduce GC.GetTotalMemory overhead
|
||||||
@@ -252,12 +252,19 @@ public class BacktestExecutor
|
|||||||
// Check for cancellation (timeout or shutdown)
|
// Check for cancellation (timeout or shutdown)
|
||||||
cancellationToken.ThrowIfCancellationRequested();
|
cancellationToken.ThrowIfCancellationRequested();
|
||||||
|
|
||||||
// Add to HashSet for reuse
|
// Maintain rolling window of last 600 candles to prevent exponential memory growth
|
||||||
fixedCandles.Add(candle);
|
rollingWindowCandles.Enqueue(candle);
|
||||||
|
if (rollingWindowCandles.Count > RollingWindowSize)
|
||||||
|
{
|
||||||
|
rollingWindowCandles.Dequeue(); // Remove oldest candle
|
||||||
|
}
|
||||||
|
|
||||||
tradingBot.LastCandle = candle;
|
tradingBot.LastCandle = candle;
|
||||||
|
|
||||||
// Run with optimized backtest path (minimize async calls)
|
// Run with optimized backtest path (minimize async calls)
|
||||||
var signalUpdateStart = Stopwatch.GetTimestamp();
|
var signalUpdateStart = Stopwatch.GetTimestamp();
|
||||||
|
// Convert rolling window to HashSet for TradingBot.UpdateSignals compatibility
|
||||||
|
var fixedCandles = new HashSet<Candle>(rollingWindowCandles);
|
||||||
await tradingBot.UpdateSignals(fixedCandles, preCalculatedIndicatorValues);
|
await tradingBot.UpdateSignals(fixedCandles, preCalculatedIndicatorValues);
|
||||||
signalUpdateTotalTime += Stopwatch.GetElapsedTime(signalUpdateStart);
|
signalUpdateTotalTime += Stopwatch.GetElapsedTime(signalUpdateStart);
|
||||||
|
|
||||||
|
|||||||
@@ -18,3 +18,5 @@ DateTime,TestName,CandlesCount,ExecutionTimeSeconds,ProcessingRateCandlesPerSec,
|
|||||||
2025-11-14T12:43:33Z,Telemetry_ETH_RSI_EMACROSS,5760,6.43,896.2,29.02,22.13,36.21,0.0,0,0.0,0.0,0.0,0.0,-35450.45,20,-49.76,0.00,b60295fc,dev,development
|
2025-11-14T12:43:33Z,Telemetry_ETH_RSI_EMACROSS,5760,6.43,896.2,29.02,22.13,36.21,0.0,0,0.0,0.0,0.0,0.0,-35450.45,20,-49.76,0.00,b60295fc,dev,development
|
||||||
2025-11-14T12:45:51Z,Telemetry_ETH_RSI_EMACROSS,5760,4.12,1398.2,29.05,21.88,36.24,0.0,0,0.0,0.0,0.0,0.0,-35450.45,20,-49.76,0.00,b60295fc,dev,development
|
2025-11-14T12:45:51Z,Telemetry_ETH_RSI_EMACROSS,5760,4.12,1398.2,29.05,21.88,36.24,0.0,0,0.0,0.0,0.0,0.0,-35450.45,20,-49.76,0.00,b60295fc,dev,development
|
||||||
2025-11-14T12:46:43Z,Telemetry_ETH_RSI_EMACROSS,5760,3.86,1491.9,29.05,20.88,36.27,0.0,0,0.0,0.0,0.0,0.0,-35450.45,20,-49.76,0.00,b60295fc,dev,development
|
2025-11-14T12:46:43Z,Telemetry_ETH_RSI_EMACROSS,5760,3.86,1491.9,29.05,20.88,36.27,0.0,0,0.0,0.0,0.0,0.0,-35450.45,20,-49.76,0.00,b60295fc,dev,development
|
||||||
|
2025-11-15T06:46:21Z,Telemetry_ETH_RSI_EMACROSS,5760,12.58,457.8,28.82,21.79,35.28,0.0,0,0.0,0.0,0.0,0.0,-35450.45,20,-49.76,0.00,e814eb74,dev,development
|
||||||
|
2025-11-15T06:50:04Z,Telemetry_ETH_RSI_EMACROSS,5760,4.84,1190.4,29.01,19.10,35.17,0.0,0,0.0,0.0,0.0,0.0,-35450.45,20,-49.76,0.00,e814eb74,dev,development
|
||||||
|
|||||||
|
@@ -63,3 +63,5 @@ DateTime,TestName,CandlesCount,ExecutionTimeSeconds,ProcessingRateCandlesPerSec,
|
|||||||
2025-11-14T12:43:33Z,Telemetry_ETH_RSI,5760,4.345,1324.3,21.03,20.35,28.95,0.00,0,0.0,4210.13,0.00,0.73,-30689.97,24,-51.70,0.00,b60295fc,dev,development
|
2025-11-14T12:43:33Z,Telemetry_ETH_RSI,5760,4.345,1324.3,21.03,20.35,28.95,0.00,0,0.0,4210.13,0.00,0.73,-30689.97,24,-51.70,0.00,b60295fc,dev,development
|
||||||
2025-11-14T12:45:51Z,Telemetry_ETH_RSI,5760,2.235,2576.6,29.03,20.37,36.11,0.00,0,0.0,2103.00,0.00,0.37,-30689.97,24,-51.70,0.00,b60295fc,dev,development
|
2025-11-14T12:45:51Z,Telemetry_ETH_RSI,5760,2.235,2576.6,29.03,20.37,36.11,0.00,0,0.0,2103.00,0.00,0.37,-30689.97,24,-51.70,0.00,b60295fc,dev,development
|
||||||
2025-11-14T12:46:43Z,Telemetry_ETH_RSI,5760,3.44,1669.8,28.85,20.34,35.90,0.00,0,0.0,3304.34,0.00,0.57,-30689.97,24,-51.70,0.00,b60295fc,dev,development
|
2025-11-14T12:46:43Z,Telemetry_ETH_RSI,5760,3.44,1669.8,28.85,20.34,35.90,0.00,0,0.0,3304.34,0.00,0.57,-30689.97,24,-51.70,0.00,b60295fc,dev,development
|
||||||
|
2025-11-15T06:46:21Z,Telemetry_ETH_RSI,5760,4.83,1191.0,29.02,20.22,37.20,4105.51,0,0.0,499.39,0.00,0.09,-30689.97,24,-51.70,0.00,e814eb74,dev,development
|
||||||
|
2025-11-15T06:50:04Z,Telemetry_ETH_RSI,5760,4.47,1286.2,28.81,20.58,34.89,3324.75,0,0.0,965.71,0.00,0.17,-30689.97,24,-51.70,0.00,e814eb74,dev,development
|
||||||
|
|||||||
|
Reference in New Issue
Block a user