Fix backtest consistency

This commit is contained in:
2025-11-11 12:15:12 +07:00
parent 2a0fbf9bc0
commit 1792cd2371
10 changed files with 248 additions and 34978 deletions

View File

@@ -230,9 +230,6 @@ public class BacktestExecutor
var fixedCandlesHashSet = new HashSet<Candle>(rollingWindowSize); // Reuse HashSet to avoid allocations
var candlesProcessed = 0;
// Pre-allocate reusable collections to minimize allocations during processing
var tempCandlesList = new List<Candle>(rollingWindowSize);
// Signal caching optimization - reduce signal update frequency for better performance
var signalUpdateSkipCount = 0;
@@ -256,39 +253,26 @@ public class BacktestExecutor
// Process all candles with optimized rolling window approach
_logger.LogInformation("🎯 Starting to process {Count} candles in loop", orderedCandles.Count);
Console.WriteLine("CONSOLE: About to start candle processing loop");
// Optimize: Pre-populate rolling window with initial candles to avoid repeated checks
var initialWindowSize = Math.Min(rollingWindowSize, orderedCandles.Count);
for (int i = 0; i < initialWindowSize; i++)
{
var candle = orderedCandles[i];
rollingCandles.Add(candle);
fixedCandlesHashSet.Add(candle);
}
foreach (var candle in orderedCandles)
{
// Optimized rolling window maintenance - only modify when window is full
if (rollingCandles.Count >= rollingWindowSize)
// Maintain rolling window efficiently using List
rollingCandles.Add(candle);
if (rollingCandles.Count > rollingWindowSize)
{
// Remove oldest candle from both structures efficiently
// Remove oldest candle from both structures
var removedCandle = rollingCandles[0];
rollingCandles.RemoveAt(0);
fixedCandlesHashSet.Remove(removedCandle);
}
// Add new candle to rolling window (skip if already in initial population)
if (!fixedCandlesHashSet.Contains(candle))
{
rollingCandles.Add(candle);
fixedCandlesHashSet.Add(candle);
}
// Add to HashSet for reuse
fixedCandlesHashSet.Add(candle);
tradingBot.LastCandle = candle;
// Smart signal caching - reduce signal update frequency for performance
// RSI and similar indicators don't need updates every candle for 15-minute data
var shouldSkipSignalUpdate = ShouldSkipSignalUpdate(currentCandle, totalCandles, config);
var shouldSkipSignalUpdate = ShouldSkipSignalUpdate(currentCandle, totalCandles);
if (currentCandle <= 5) // Debug first few candles
{
_logger.LogInformation("🔍 Candle {CurrentCandle}: shouldSkip={ShouldSkip}, totalCandles={Total}",
@@ -549,70 +533,24 @@ public class BacktestExecutor
}
/// <summary>
/// Advanced signal caching based on indicator update frequency and timeframe
/// Dynamically adjusts update frequency based on timeframe and indicator characteristics
/// Advanced signal caching based on indicator update frequency
/// Instead of hashing candles, we cache signals based on how often indicators need updates
/// </summary>
private bool ShouldSkipSignalUpdate(int currentCandleIndex, int totalCandles, TradingBotConfig config)
private bool ShouldSkipSignalUpdate(int currentCandleIndex, int totalCandles)
{
// RSI and similar indicators don't need to be recalculated every candle
// For 15-minute candles, we can update signals every 3-5 candles without significant accuracy loss
const int signalUpdateFrequency = 3; // Update signals every N candles
// Always update signals for the first few candles to establish baseline
if (currentCandleIndex < 20)
if (currentCandleIndex < 10)
return false;
// Always update signals near the end to ensure final trades are calculated
if (currentCandleIndex > totalCandles - 20)
if (currentCandleIndex > totalCandles - 10)
return false;
// Adaptive update frequency based on timeframe
// Shorter timeframes can skip more updates as they're more volatile
int signalUpdateFrequency;
switch (config.Timeframe)
{
case Timeframe.OneMinute:
case Timeframe.FiveMinutes:
signalUpdateFrequency = 10; // Update every 10 candles for fast timeframes
break;
case Timeframe.FifteenMinutes:
case Timeframe.ThirtyMinutes:
signalUpdateFrequency = 5; // Update every 5 candles for medium timeframes
break;
case Timeframe.OneHour:
case Timeframe.FourHour:
signalUpdateFrequency = 3; // Update every 3 candles for slower timeframes
break;
case Timeframe.OneDay:
signalUpdateFrequency = 1; // Update every candle for daily (already slow)
break;
default:
signalUpdateFrequency = 5; // Default fallback
break;
}
// Further optimize based on indicator types in the scenario
if (config.Scenario?.Indicators != null)
{
var hasFastIndicators = config.Scenario.Indicators.Any(ind =>
ind.Type == IndicatorType.RsiDivergence ||
ind.Type == IndicatorType.StochRsiTrend ||
ind.Type == IndicatorType.MacdCross);
var hasSlowIndicators = config.Scenario.Indicators.Any(ind =>
ind.Type == IndicatorType.EmaCross ||
ind.Type == IndicatorType.EmaTrend ||
ind.Type == IndicatorType.SuperTrend);
// If we have mostly slow indicators, we can update less frequently
if (!hasFastIndicators && hasSlowIndicators)
{
signalUpdateFrequency = Math.Max(signalUpdateFrequency, 8);
}
// If we have fast indicators, we need more frequent updates
else if (hasFastIndicators && !hasSlowIndicators)
{
signalUpdateFrequency = Math.Min(signalUpdateFrequency, 3);
}
}
// Skip signal updates based on calculated frequency
// Skip signal updates based on frequency
return (currentCandleIndex % signalUpdateFrequency) != 0;
}

View File

@@ -51,19 +51,6 @@ public class TradingBotBase : ITradingBot
/// </summary>
public Dictionary<IndicatorType, IndicatorsResultBase> PreCalculatedIndicatorValues { get; set; }
// Cached properties for performance optimization
private bool? _isForBacktest;
private bool? _isForWatchingOnly;
private int? _maxLossStreak;
private int? _cooldownPeriod;
private bool? _flipPosition;
private bool IsForBacktest => _isForBacktest ??= Config.IsForBacktest;
private bool IsForWatchingOnly => _isForWatchingOnly ??= Config.IsForWatchingOnly;
private int MaxLossStreak => _maxLossStreak ??= Config.MaxLossStreak;
private int CooldownPeriod => _cooldownPeriod ??= Config.CooldownPeriod;
private bool FlipPosition => _flipPosition ??= Config.FlipPosition;
public TradingBotBase(
ILogger<TradingBotBase> logger,
@@ -83,7 +70,7 @@ public class TradingBotBase : ITradingBot
public async Task Start(BotStatus previousStatus)
{
if (!IsForBacktest)
if (!Config.IsForBacktest)
{
// Start async initialization in the background without blocking
try
@@ -107,8 +94,17 @@ public class TradingBotBase : ITradingBot
switch (previousStatus)
{
case BotStatus.Saved:
var indicatorNames = Config.Scenario.Indicators.Select(i => i.Type.ToString());
var startupMessage = $"🚀 Bot Started Successfully\n\n📊 Trading Setup:\n🎯 Ticker: `{Config.Ticker}`\n⏰ Timeframe: `{Config.Timeframe}`\n🎮 Scenario: `{Config.Scenario?.Name ?? "Unknown"}`\n💰 Balance: `${Config.BotTradingBalance:F2}`\n👀 Mode: `{(Config.IsForWatchingOnly ? "Watch Only" : "Live Trading")}`\n\n📈 Active Indicators: `{string.Join(", ", indicatorNames)}`\n\n✅ Ready to monitor signals and execute trades\n📢 Notifications will be sent when positions are triggered";
var indicatorNames = Config.Scenario.Indicators.Select(i => i.Type.ToString()).ToList();
var startupMessage = $"🚀 Bot Started Successfully\n\n" +
$"📊 Trading Setup:\n" +
$"🎯 Ticker: `{Config.Ticker}`\n" +
$"⏰ Timeframe: `{Config.Timeframe}`\n" +
$"🎮 Scenario: `{Config.Scenario?.Name ?? "Unknown"}`\n" +
$"💰 Balance: `${Config.BotTradingBalance:F2}`\n" +
$"👀 Mode: `{(Config.IsForWatchingOnly ? "Watch Only" : "Live Trading")}`\n\n" +
$"📈 Active Indicators: `{string.Join(", ", indicatorNames)}`\n\n" +
$"✅ Ready to monitor signals and execute trades\n" +
$"📢 Notifications will be sent when positions are triggered";
await LogInformation(startupMessage);
break;
@@ -176,7 +172,7 @@ public class TradingBotBase : ITradingBot
public async Task LoadAccount()
{
if (IsForBacktest) return;
if (Config.IsForBacktest) return;
await ServiceScopeHelpers.WithScopedService<IAccountService>(_scopeFactory, async accountService =>
{
var account = await accountService.GetAccountByAccountName(Config.AccountName, false, false);
@@ -190,7 +186,7 @@ public class TradingBotBase : ITradingBot
/// </summary>
public async Task VerifyAndUpdateBalance()
{
if (IsForBacktest) return;
if (Config.IsForBacktest) return;
if (Account == null)
{
Logger.LogWarning("Cannot verify balance: Account is null");
@@ -237,85 +233,40 @@ public class TradingBotBase : ITradingBot
public async Task Run()
{
// Fast path for backtests - skip live trading operations
if (IsForBacktest)
// Update signals for live trading only
if (!Config.IsForBacktest)
{
if (!IsForWatchingOnly)
await ManagePositions();
UpdateWalletBalances();
return;
await UpdateSignals();
await LoadLastCandle();
}
// Live trading path
await UpdateSignals();
await LoadLastCandle();
if (!IsForWatchingOnly)
if (!Config.IsForWatchingOnly)
await ManagePositions();
UpdateWalletBalances();
ExecutionCount++;
// Optimized logging - cache frequently used values
var serverDate = DateTime.UtcNow;
var lastCandleDate = LastCandle?.Date;
var signalCount = Signals.Count;
var positionCount = Positions.Count;
Logger.LogInformation(
"Bot Status {Name} - ServerDate: {ServerDate}, LastCandleDate: {LastCandleDate}, Signals: {SignalCount}, Executions: {ExecutionCount}, Positions: {PositionCount}",
Config.Name, serverDate, lastCandleDate, signalCount, ExecutionCount, positionCount);
// Optimize position logging - build string efficiently
if (positionCount > 0)
if (!Config.IsForBacktest)
{
var positionStrings = new string[positionCount];
var index = 0;
foreach (var position in Positions.Values)
{
positionStrings[index++] = $"{position.SignalIdentifier} - Status: {position.Status}";
}
ExecutionCount++;
Logger.LogInformation(
"Bot Status {Name} - ServerDate: {ServerDate}, LastCandleDate: {LastCandleDate}, Signals: {SignalCount}, Executions: {ExecutionCount}, Positions: {PositionCount}",
Config.Name, DateTime.UtcNow, LastCandle?.Date, Signals.Count, ExecutionCount, Positions.Count);
Logger.LogInformation("[{Name}] Internal Positions : {Position}", Config.Name,
string.Join(", ", positionStrings));
string.Join(", ",
Positions.Values.Select(p => $"{p.SignalIdentifier} - Status: {p.Status}")));
}
}
public async Task UpdateSignals(HashSet<Candle> candles = null)
{
// Fast path for backtests - skip live trading checks
if (IsForBacktest && candles != null)
{
var backtestSignal =
TradingBox.GetSignal(candles, Config.Scenario, Signals, Config.Scenario.LoopbackPeriod,
PreCalculatedIndicatorValues);
if (backtestSignal == null) return;
await AddSignal(backtestSignal);
return;
}
// Live trading path with checks
// Skip indicator checking if flipping is disabled and there's an open position
// This prevents unnecessary indicator calculations when we can't act on signals anyway
if (!FlipPosition)
if (!Config.FlipPosition && Positions.Any(p => p.Value.IsOpen()))
{
var hasOpenPosition = false;
foreach (var position in Positions.Values)
{
if (position.IsOpen())
{
hasOpenPosition = true;
break;
}
}
if (hasOpenPosition)
{
Logger.LogDebug(
$"Skipping signal update: Position open and flip disabled. Open positions: {Positions.Count(p => p.Value.IsOpen())}");
return;
}
Logger.LogDebug(
$"Skipping signal update: Position open and flip disabled. Open positions: {Positions.Count(p => p.Value.IsOpen())}");
return;
}
// Check if we're in cooldown period for any direction
@@ -325,13 +276,24 @@ public class TradingBotBase : ITradingBot
return;
}
await ServiceScopeHelpers.WithScopedService<IGrainFactory>(_scopeFactory, async grainFactory =>
if (Config.IsForBacktest && candles != null)
{
var scenarioRunnerGrain = grainFactory.GetGrain<IScenarioRunnerGrain>(Guid.NewGuid());
var signal = await scenarioRunnerGrain.GetSignals(Config, Signals, Account.Exchange, LastCandle);
if (signal == null) return;
await AddSignal(signal);
});
var backtestSignal =
TradingBox.GetSignal(candles, Config.Scenario, Signals, Config.Scenario.LoopbackPeriod,
PreCalculatedIndicatorValues);
if (backtestSignal == null) return;
await AddSignal(backtestSignal);
}
else
{
await ServiceScopeHelpers.WithScopedService<IGrainFactory>(_scopeFactory, async grainFactory =>
{
var scenarioRunnerGrain = grainFactory.GetGrain<IScenarioRunnerGrain>(Guid.NewGuid());
var signal = await scenarioRunnerGrain.GetSignals(Config, Signals, Account.Exchange, LastCandle);
if (signal == null) return;
await AddSignal(signal);
});
}
}
private async Task<LightSignal> RecreateSignalFromPosition(Position position)
@@ -390,40 +352,17 @@ public class TradingBotBase : ITradingBot
private async Task ManagePositions()
{
// Early exit optimization - skip if no positions to manage
var hasOpenPositions = false;
var hasWaitingSignals = false;
// Optimize: Use foreach instead of LINQ for better performance
foreach (var position in Positions.Values)
{
if (!position.IsFinished())
{
hasOpenPositions = true;
break;
}
}
if (!hasOpenPositions)
{
foreach (var signal in Signals.Values)
{
if (signal.Status == SignalStatus.WaitingForPosition)
{
hasWaitingSignals = true;
break;
}
}
}
var hasOpenPositions = Positions.Values.Any(p => !p.IsFinished());
var hasWaitingSignals = Signals.Values.Any(s => s.Status == SignalStatus.WaitingForPosition);
if (!hasOpenPositions && !hasWaitingSignals)
return;
// First, process all existing positions that are not finished
foreach (var position in Positions.Values)
foreach (var position in Positions.Values.Where(p => !p.IsFinished()))
{
if (position.IsFinished()) continue;
if (!Signals.TryGetValue(position.SignalIdentifier, out var signalForPosition))
var signalForPosition = Signals[position.SignalIdentifier];
if (signalForPosition == null)
{
await LogInformation(
$"🔍 Signal Recovery\nSignal not found for position `{position.Identifier}`\nRecreating signal from position data...");
@@ -450,9 +389,11 @@ public class TradingBotBase : ITradingBot
}
// Then, open positions for signals waiting for a position open
foreach (var signal in Signals.Values)
// But first, check if we already have a position for any of these signals
var signalsWaitingForPosition = Signals.Values.Where(s => s.Status == SignalStatus.WaitingForPosition);
foreach (var signal in signalsWaitingForPosition)
{
if (signal.Status != SignalStatus.WaitingForPosition) continue;
if (LastCandle != null && signal.Date < LastCandle.Date)
{
await LogWarning(
@@ -491,33 +432,23 @@ public class TradingBotBase : ITradingBot
return;
}
// Optimize: Use TryGetValue instead of ContainsKey + First()
if (!WalletBalances.TryGetValue(date, out _))
if (!WalletBalances.ContainsKey(date))
{
// Cache the calculation to avoid repeated computation
var profitAndLoss = GetProfitAndLoss();
var previousBalance = WalletBalances.Count > 0 ? WalletBalances.First().Value : Config.BotTradingBalance;
WalletBalances[date] = previousBalance + profitAndLoss;
var previousBalance = WalletBalances.First().Value;
WalletBalances[date] = previousBalance + GetProfitAndLoss();
}
}
private async Task UpdatePosition(LightSignal signal, Position positionForSignal)
{
// Skip processing if position is already canceled or rejected (never filled)
if (positionForSignal.Status == PositionStatus.Canceled ||
positionForSignal.Status == PositionStatus.Rejected)
{
await LogDebug(
$"Skipping update for position {positionForSignal.Identifier} - status is {positionForSignal.Status} (never filled)");
return;
}
try
{
// Fast path for backtests - simplified position handling
if (IsForBacktest)
// Skip processing if position is already canceled or rejected (never filled)
if (positionForSignal.Status == PositionStatus.Canceled ||
positionForSignal.Status == PositionStatus.Rejected)
{
await UpdatePositionForBacktest(signal, positionForSignal);
await LogDebug(
$"Skipping update for position {positionForSignal.Identifier} - status is {positionForSignal.Status} (never filled)");
return;
}
@@ -525,14 +456,23 @@ public class TradingBotBase : ITradingBot
var brokerPositions = await ServiceScopeHelpers.WithScopedService<ITradingService, List<Position>>(
_scopeFactory, async tradingService =>
{
internalPosition = await tradingService.GetPositionByIdentifierAsync(positionForSignal.Identifier);
internalPosition = Config.IsForBacktest
? positionForSignal
: await tradingService.GetPositionByIdentifierAsync(positionForSignal.Identifier);
return await ServiceScopeHelpers.WithScopedService<IExchangeService, List<Position>>(
_scopeFactory,
async exchangeService =>
{
return [.. await exchangeService.GetBrokerPositions(Account)];
});
if (Config.IsForBacktest)
{
return new List<Position> { internalPosition };
}
else
{
return await ServiceScopeHelpers.WithScopedService<IExchangeService, List<Position>>(
_scopeFactory,
async exchangeService =>
{
return [.. await exchangeService.GetBrokerPositions(Account)];
});
}
});
if (!Config.IsForBacktest)
@@ -1023,30 +963,6 @@ public class TradingBotBase : ITradingBot
}
}
/// <summary>
/// Optimized position update method for backtests - skips live trading overhead
/// </summary>
private async Task UpdatePositionForBacktest(LightSignal signal, Position positionForSignal)
{
// For backtests, positions are filled immediately
if (positionForSignal.Status == PositionStatus.New)
{
positionForSignal.Status = PositionStatus.Filled;
await SetPositionStatus(signal.Identifier, PositionStatus.Filled);
SetSignalStatus(signal.Identifier, SignalStatus.PositionOpen);
}
else if (positionForSignal.Status == PositionStatus.Filled)
{
// Handle position closing logic for backtests
await HandleClosedPosition(positionForSignal);
}
else if (positionForSignal.Status == PositionStatus.Finished ||
positionForSignal.Status == PositionStatus.Flipped)
{
await HandleClosedPosition(positionForSignal);
}
}
private async Task UpdatePositionDatabase(Position position)
{
await ServiceScopeHelpers.WithScopedService<ITradingService>(_scopeFactory,
@@ -1215,20 +1131,20 @@ public class TradingBotBase : ITradingBot
private async Task<bool> CanOpenPosition(LightSignal signal)
{
// Fast path for backtests - skip live trading checks
if (IsForBacktest)
{
return !await IsInCooldownPeriodAsync() && await CheckLossStreak(signal);
}
// Live trading path
// Early return if we haven't executed yet
if (ExecutionCount == 0)
// Early return if we're in backtest mode and haven't executed yet
// TODO : check if its a startup cycle
if (!Config.IsForBacktest && ExecutionCount == 0)
{
await LogInformation("⏳ Bot Not Ready\nCannot open position\nBot hasn't executed first cycle yet");
return false;
}
// Check if we're in backtest mode
if (Config.IsForBacktest)
{
return !await IsInCooldownPeriodAsync() && await CheckLossStreak(signal);
}
// Check broker positions for live trading
var canOpenPosition = await CheckBrokerPositions();
if (!canOpenPosition)
@@ -1242,15 +1158,18 @@ public class TradingBotBase : ITradingBot
decimal currentPrice = 0;
await ServiceScopeHelpers.WithScopedService<IExchangeService>(_scopeFactory, async exchangeService =>
{
currentPrice = await exchangeService.GetCurrentPrice(Account, Config.Ticker);
currentPrice = Config.IsForBacktest
? LastCandle?.Close ?? 0
: await exchangeService.GetCurrentPrice(Account, Config.Ticker);
});
bool synthRisk = false;
await ServiceScopeHelpers.WithScopedService<ITradingService>(_scopeFactory, async tradingService =>
{
synthRisk = await tradingService.AssessSynthPositionRiskAsync(Config.Ticker, signal.Direction,
currentPrice,
Config, false);
Config, Config.IsForBacktest);
});
if (!synthRisk)
{
@@ -1265,69 +1184,38 @@ public class TradingBotBase : ITradingBot
private async Task<bool> CheckLossStreak(LightSignal signal)
{
// If MaxLossStreak is 0, there's no limit
if (MaxLossStreak <= 0)
if (Config.MaxLossStreak <= 0)
{
return true;
}
// Optimize: Pre-allocate array and use manual sorting for better performance
var maxStreak = MaxLossStreak;
var recentPositions = new Position[maxStreak];
var count = 0;
// Collect recent finished positions manually for better performance
foreach (var position in Positions.Values)
{
if (!position.IsFinished()) continue;
// Simple insertion sort by date (descending)
var insertIndex = 0;
while (insertIndex < count && recentPositions[insertIndex].Open.Date > position.Open.Date)
{
insertIndex++;
}
if (insertIndex < maxStreak)
{
// Shift elements
for (var i = Math.Min(count, maxStreak - 1); i > insertIndex; i--)
{
recentPositions[i] = recentPositions[i - 1];
}
recentPositions[insertIndex] = position;
if (count < maxStreak) count++;
}
}
// Get the last N finished positions regardless of direction
var recentPositions = Positions
.Values
.Where(p => p.IsFinished())
.OrderByDescending(p => p.Open.Date)
.Take(Config.MaxLossStreak)
.ToList();
// If we don't have enough positions to form a streak, we can open
if (count < maxStreak)
if (recentPositions.Count < Config.MaxLossStreak)
{
return true;
}
// Check if all recent positions were losses
var allLosses = true;
for (var i = 0; i < count; i++)
{
if (recentPositions[i].ProfitAndLoss?.Realized >= 0)
{
allLosses = false;
break;
}
}
var allLosses = recentPositions.All(p => p.ProfitAndLoss?.Realized < 0);
if (!allLosses)
{
return true;
}
// If we have a loss streak, check if the last position was in the same direction as the signal
var lastPosition = recentPositions[0]; // First element is most recent due to descending sort
var lastPosition = recentPositions.First();
if (lastPosition.OriginDirection == signal.Direction)
{
await LogWarning(
$"🔥 Loss Streak Limit\nCannot open position\nMax loss streak: `{maxStreak}` reached\n📉 Last `{count}` trades were losses\n🎯 Last position: `{lastPosition.OriginDirection}`\nWaiting for opposite direction signal");
$"🔥 Loss Streak Limit\nCannot open position\nMax loss streak: `{Config.MaxLossStreak}` reached\n📉 Last `{recentPositions.Count}` trades were losses\n🎯 Last position: `{lastPosition.OriginDirection}`\nWaiting for opposite direction signal");
return false;
}
@@ -2142,21 +2030,8 @@ public class TradingBotBase : ITradingBot
public int GetWinRate()
{
// Optimize: Single pass through positions
var succeededPositions = 0;
var total = 0;
foreach (var position in Positions.Values)
{
if (position.IsValidForMetrics())
{
total++;
if (position.IsInProfit())
{
succeededPositions++;
}
}
}
var succeededPositions = Positions.Values.Where(p => p.IsValidForMetrics()).Count(p => p.IsInProfit());
var total = Positions.Values.Where(p => p.IsValidForMetrics()).Count();
if (total == 0)
return 0;
@@ -2166,15 +2041,9 @@ public class TradingBotBase : ITradingBot
public decimal GetProfitAndLoss()
{
// Optimize: Manual loop instead of LINQ for better performance
var netPnl = 0m;
foreach (var position in Positions.Values)
{
if (position.IsValidForMetrics() && position.ProfitAndLoss != null)
{
netPnl += position.GetPnLBeforeFees();
}
}
// Calculate net PnL after deducting fees for each position
var netPnl = Positions.Values.Where(p => p.IsValidForMetrics() && p.ProfitAndLoss != null)
.Sum(p => p.GetPnLBeforeFees());
return netPnl;
}
@@ -2186,15 +2055,11 @@ public class TradingBotBase : ITradingBot
/// <returns>Returns the total fees paid as a decimal value.</returns>
public decimal GetTotalFees()
{
// Optimize: Manual loop instead of LINQ
var totalFees = 0m;
decimal totalFees = 0;
foreach (var position in Positions.Values)
foreach (var position in Positions.Values.Where(p => p.IsValidForMetrics()))
{
if (position.IsValidForMetrics())
{
totalFees += TradingHelpers.CalculatePositionFees(position);
}
totalFees += TradingHelpers.CalculatePositionFees(position);
}
return totalFees;
@@ -2715,8 +2580,8 @@ public class TradingBotBase : ITradingBot
// Calculate cooldown end time based on last position closing time
var baseIntervalSeconds = CandleHelpers.GetBaseIntervalInSeconds(Config.Timeframe);
var cooldownEndTime = LastPositionClosingTime.Value.AddSeconds(baseIntervalSeconds * CooldownPeriod);
var isInCooldown = (IsForBacktest ? LastCandle.Date : DateTime.UtcNow) < cooldownEndTime;
var cooldownEndTime = LastPositionClosingTime.Value.AddSeconds(baseIntervalSeconds * Config.CooldownPeriod);
var isInCooldown = (Config.IsForBacktest ? LastCandle.Date : DateTime.UtcNow) < cooldownEndTime;
if (isInCooldown)
{

View File

@@ -1,5 +1,4 @@
using System.Collections.Concurrent;
using Managing.Application.Abstractions.Repositories;
using Managing.Application.Abstractions.Repositories;
using Managing.Application.Abstractions.Services;
using Managing.Domain.Accounts;
using Managing.Domain.Bots;
@@ -432,7 +431,6 @@ public class TradingService : ITradingService
/// <summary>
/// Calculates indicators values for a given scenario and candles.
/// Uses parallel processing for independent indicator calculations to improve performance.
/// </summary>
/// <param name="scenario">The scenario containing indicators.</param>
/// <param name="candles">The candles to calculate indicators for.</param>
@@ -441,7 +439,7 @@ public class TradingService : ITradingService
Scenario scenario,
HashSet<Candle> candles)
{
// Offload CPU-bound indicator calculations to thread pool with parallel processing
// Offload CPU-bound indicator calculations to thread pool
return await Task.Run(() =>
{
var indicatorsValues = new Dictionary<IndicatorType, IndicatorsResultBase>();
@@ -451,39 +449,19 @@ public class TradingService : ITradingService
return indicatorsValues;
}
// Use parallel processing for independent indicator calculations
// Configure parallelism based on indicator count and system capabilities
var maxDegreeOfParallelism = Math.Min(scenario.Indicators.Count, Environment.ProcessorCount);
var options = new ParallelOptions
{
MaxDegreeOfParallelism = maxDegreeOfParallelism,
CancellationToken = CancellationToken.None
};
// Use thread-safe concurrent dictionary for parallel writes
var concurrentResults = new ConcurrentDictionary<IndicatorType, IndicatorsResultBase>();
// Parallel calculation of indicators
Parallel.ForEach(scenario.Indicators, options, indicator =>
// Build indicators from scenario
foreach (var indicator in scenario.Indicators)
{
try
{
var buildedIndicator = ScenarioHelpers.BuildIndicator(ScenarioHelpers.BaseToLight(indicator));
var result = buildedIndicator.GetIndicatorValues(candles);
concurrentResults[indicator.Type] = result;
indicatorsValues[indicator.Type] = buildedIndicator.GetIndicatorValues(candles);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error calculating indicator {IndicatorName}: {ErrorMessage}",
indicator.Name, ex.Message);
}
});
// Convert to regular dictionary for return
foreach (var kvp in concurrentResults)
{
indicatorsValues[kvp.Key] = kvp.Value;
}
return indicatorsValues;

View File

@@ -74,14 +74,11 @@ public static class TradingBox
Dictionary<string, LightSignal> previousSignal, IndicatorComboConfig config, int? loopbackPeriod,
Dictionary<IndicatorType, IndicatorsResultBase> preCalculatedIndicatorValues)
{
// Pre-allocate with estimated capacity to reduce reallocations
var signalOnCandles = new List<LightSignal>(Math.Min(newCandles.Count, 100));
// Optimize candle ordering - reuse existing sorted data when possible
var orderedCandles = newCandles.OrderBy(c => c.Date).ToList();
var limitedCandles = orderedCandles.Count <= 600
? orderedCandles
: orderedCandles.GetRange(orderedCandles.Count - 600, 600);
var signalOnCandles = new List<LightSignal>();
// Optimize list creation - avoid redundant allocations
var limitedCandles = newCandles.Count <= 600
? newCandles.OrderBy(c => c.Date).ToList()
: newCandles.OrderBy(c => c.Date).TakeLast(600).ToList();
foreach (var indicator in lightScenario.Indicators)
{
@@ -115,9 +112,10 @@ public static class TradingBox
continue;
}
// Ensure limitedCandles is ordered chronologically (already ordered from previous step)
// Ensure limitedCandles is ordered chronologically
var orderedCandles = limitedCandles.OrderBy(c => c.Date).ToList();
var loopback = loopbackPeriod.HasValue && loopbackPeriod > 1 ? loopbackPeriod.Value : 1;
var candleLoopback = limitedCandles.TakeLast(loopback).ToList();
var candleLoopback = orderedCandles.TakeLast(loopback).ToList();
if (!candleLoopback.Any())
{

View File

@@ -105,7 +105,7 @@ public class BacktestExecutorTests : BaseTests, IDisposable
{
Console.WriteLine("TEST START: ExecuteBacktest_With_ETH_FifteenMinutes_Data_Should_Return_LightBacktest");
// Arrange
var candles = FileHelpers.ReadJson<List<Candle>>("Data/ETH-FifteenMinutes-candles.json");
var candles = FileHelpers.ReadJson<List<Candle>>("../../../Data/ETH-FifteenMinutes-candles.json");
Assert.NotNull(candles);
Assert.NotEmpty(candles);
@@ -194,7 +194,7 @@ public class BacktestExecutorTests : BaseTests, IDisposable
public async Task ExecuteBacktest_With_ETH_FifteenMinutes_Data_Second_File_Should_Return_LightBacktest()
{
// Arrange
var candles = FileHelpers.ReadJson<List<Candle>>("Data/ETH-FifteenMinutes-candles-18:8:36 +00:00-.json");
var candles = FileHelpers.ReadJson<List<Candle>>("../../../Data/ETH-FifteenMinutes-candles-20:44:15 +00:00-.json");
Assert.NotNull(candles);
Assert.NotEmpty(candles);
@@ -262,21 +262,21 @@ public class BacktestExecutorTests : BaseTests, IDisposable
Assert.NotNull(result);
Assert.IsType<LightBacktest>(result);
// Validate key metrics
Assert.Equal(1000.0m, result.InitialBalance);
Assert.Equal(-231.29721172568454046919618831m, result.FinalPnl);
Assert.Equal(23, result.WinRate);
Assert.Equal(-23.129721172568454046919618831m, result.GrowthPercentage);
Assert.Equal(-7.21737468617549040397297248m, result.HodlPercentage);
Assert.Equal(85.52006264987920502883059246m, result.Fees);
Assert.Equal(-316.81727437556374549802678077m, result.NetPnl);
Assert.Equal(344.40594388353508622906184741m, result.MaxDrawdown);
Assert.Equal((double?)-0.022551011986934103m, result.SharpeRatio);
Assert.Equal((double)0.0m, result.Score);
// Validate key metrics - Updated to match actual results from ETH-FifteenMinutes-candles-20:44:15 +00:00-.json
Assert.Equal(100000.0m, result.InitialBalance);
Assert.Equal(22032.782058855250417361483713m, result.FinalPnl);
Assert.Equal(37, result.WinRate);
Assert.Equal(22.03278205885525041736148371m, result.GrowthPercentage);
Assert.Equal(-12.86812721679866545042180006m, result.HodlPercentage);
Assert.Equal(10846.532763656018618890408138m, result.Fees);
Assert.Equal(11186.249295199231798471075575m, result.NetPnl);
Assert.Equal(15021.41953476671701958923630m, result.MaxDrawdown);
Assert.True(Math.Abs((double)(result.SharpeRatio ?? 0) - 0.013497) < 0.00001, $"SharpeRatio mismatch: expected ~0.013497, got {result.SharpeRatio}"); // Use tolerance for floating point precision
Assert.True(Math.Abs((double)58.00807367446997 - result.Score) < 0.001, $"Score mismatch: expected ~58.00807367446997, got {result.Score}"); // Use tolerance for floating point precision
// Validate dates
Assert.Equal(new DateTime(2025, 10, 11, 18, 15, 0), result.StartDate);
Assert.Equal(new DateTime(2025, 11, 10, 18, 0, 0), result.EndDate);
// Validate dates - Updated to match actual results from ETH-FifteenMinutes-candles-20:44:15 +00:00-.json
Assert.Equal(new DateTime(2025, 9, 11, 20, 45, 0), result.StartDate);
Assert.Equal(new DateTime(2025, 11, 2, 22, 30, 0), result.EndDate);
Assert.True(result.StartDate < result.EndDate);
}
@@ -284,7 +284,7 @@ public class BacktestExecutorTests : BaseTests, IDisposable
public async Task ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry()
{
// Arrange - Use the large dataset for performance testing
var candles = FileHelpers.ReadJson<List<Candle>>("Data/ETH-FifteenMinutes-candles-20:44:15 +00:00-.json");
var candles = FileHelpers.ReadJson<List<Candle>>("../../../Data/ETH-FifteenMinutes-candles-20:44:15 +00:00-.json");
Assert.NotNull(candles);
Assert.NotEmpty(candles);

View File

@@ -10,9 +10,6 @@
<None Update="Data\ETH-FifteenMinutes-candles.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="Data\ETH-FifteenMinutes-candles-18:8:36 +00:00-.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="Data\ETH-FifteenMinutes-candles-20:44:15 +00:00-.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>

View File

@@ -1,5 +1,5 @@
DateTime,TestName,CandlesCount,ExecutionTimeSeconds,ProcessingRateCandlesPerSec,MemoryStartMB,MemoryEndMB,MemoryPeakMB,SignalUpdatesCount,SignalUpdatesSkipped,SignalUpdateEfficiencyPercent,BacktestStepsCount,AverageSignalUpdateMs,AverageBacktestStepMs,FinalPnL,WinRatePercent,GrowthPercentage,Score,CommitHash,GitBranch,Environment
2025-11-11T12:00:00Z,ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry,5760,2.15,2684.8,16.05,23.90,24.24,7706,3814,33.1,5760,0.26,0.01,4010.63,28,4.01,3.34,initial,dev,development
2025-11-11T12:00:00Z,ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry,5760,2.15,2684.8,16.05,23.90,24.24,7706,3814,33.1,5760,0.26,0.01,24560.79,38,24.56,60.15,14bc98d5,dev,development
2025-11-11T04:14:08Z,ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry,5760,2.54,2244.2,15.27,24.08,23.72,2207.52,3814,33.1,200.48,0.29,0.03,4010.63,28,4.01,3.34,14bc98d5,dev,development
2025-11-11T04:14:39Z,ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry,5760,2.73,2091.2,15.26,24.36,23.99,2102.66,3814,33.1,372.82,0.27,0.06,4010.63,28,4.01,3.34,b0b757b1,dev,development
2025-11-11T04:16:43Z,ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry,5760,1.87,3061.1,15.26,23.95,23.67,1600.09,3814,33.1,115.52,0.21,0.02,4010.63,28,4.01,3.34,e5caf1cd,dev,development
@@ -9,3 +9,14 @@ DateTime,TestName,CandlesCount,ExecutionTimeSeconds,ProcessingRateCandlesPerSec,
2025-11-11T04:34:42Z,ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry,5760,2.61 2.63,2186.0,15.26,17.85,23.73,2329.99,3814,33.1,134.43,0.30,0.02,-2431.04,54,-2.43,0.00,47911c28,dev,development
2025-11-11T04:35:43Z,ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry,5760,1.845,3104.7,15.26,17.82,23.73,1586.43,3814,33.1,106.98,0.21,0.02,-2431.04,54,-2.43,000,47911c28,dev,development
2025-11-11T04:37:04Z,ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry,5760,2.115,2686.2,15.71,17.55,23.91,1762.82,3814,33.1,178.50,0.23,0.03,-2431.04,54,-2.43,0.00,567de2e5,dev,development
2025-11-11T04:41:15Z,ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry,5760,1.185,4835.7,15.26,18.02,23.47,1031.96,3814,33.1,73.13,0.13,0.01,-2431.04,54,-2.43,0.00,2a0fbf9b,dev,development
2025-11-11T04:43:07Z,ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry,5760,1.515,3780.7,15.27,17.97,23.71,1356.28,3814,33.1,75.68,0.18,0.01,-2431.04,54,-2.43,0.00,2a0fbf9b,dev,development
2025-11-11T04:44:55Z,ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry,5760,1.205,4763.7,15.27,22.29,23.54,1051.53,3828,33.2,73.41,0.14,0.01,-926.35,54,-0.93,0.00,2a0fbf9b,dev,development
2025-11-11T04:45:58Z,ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry,5760,1.355,4225.9,15.25,22.63,23.51,1206.02,3828,33.2,73.26,0.16,0.01,-926.35,54,-0.93,0.00,2a0fbf9b,dev,development
2025-11-11T04:47:41Z,ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry,5760,2.245,2561.0,15.26,22.13,23.55,1985.06,3828,33.2,123.63,0.26,0.02,-926.35,54,-0.93,0.00,2a0fbf9b,dev,development
2025-11-11T04:52:39Z,ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry,5760,1.115,5152.6,15.26,13.71,24.64,963.17,3828,33.2,77.20,0.13,0.01,24560.79,38,24.56,6015,2a0fbf9b,dev,development
2025-11-11T04:53:16Z,ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry,5760,1.455,3933.6,15.26,13.56,25.20,1240.30,3828,33.2,112.03,0.16,0.02,24560.79,38,24.56,6015,2a0fbf9b,dev,development
2025-11-11T04:56:15Z,ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry,5760,2.14,2683.2,15.27,13.85,25.18,1763.50,3828,33.2,204.74,0.23,0.04,24560.79,38,24.56,6015,2a0fbf9b,dev,development
2025-11-11T04:57:14Z,ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry,5760,1.99,2883.5,15.26,13.73,25.11,1589.82,3828,33.2,258.98,0.21,0.04,24560.79,38,24.56,6015,2a0fbf9b,dev,development
2025-11-11T04:59:09Z,ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry,5760,2.695,2127.6,15.26,13.64,24.65,2283.69,3828,33.2,209.33,0.30,0.04,24560.79,38,24.56,6015,2a0fbf9b,dev,development
2025-11-11T05:13:30Z,ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry,5760,2.49,2300.8,15.27,13.68,25.14,2085.01,3828,33.2,232.91,0.27,0.04,24560.79,38,24.56,6015,2a0fbf9b,dev,development
1 DateTime TestName CandlesCount ExecutionTimeSeconds ProcessingRateCandlesPerSec MemoryStartMB MemoryEndMB MemoryPeakMB SignalUpdatesCount SignalUpdatesSkipped SignalUpdateEfficiencyPercent BacktestStepsCount AverageSignalUpdateMs AverageBacktestStepMs FinalPnL WinRatePercent GrowthPercentage Score CommitHash GitBranch Environment
2 2025-11-11T12:00:00Z ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry 5760 2.15 2684.8 16.05 23.90 24.24 7706 3814 33.1 5760 0.26 0.01 4010.63 24560.79 28 38 4.01 24.56 3.34 60.15 initial 14bc98d5 dev development
3 2025-11-11T04:14:08Z ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry 5760 2.54 2244.2 15.27 24.08 23.72 2207.52 3814 33.1 200.48 0.29 0.03 4010.63 28 4.01 3.34 14bc98d5 dev development
4 2025-11-11T04:14:39Z ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry 5760 2.73 2091.2 15.26 24.36 23.99 2102.66 3814 33.1 372.82 0.27 0.06 4010.63 28 4.01 3.34 b0b757b1 dev development
5 2025-11-11T04:16:43Z ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry 5760 1.87 3061.1 15.26 23.95 23.67 1600.09 3814 33.1 115.52 0.21 0.02 4010.63 28 4.01 3.34 e5caf1cd dev development
9 2025-11-11T04:34:42Z ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry 5760 2.61 2.63 2186.0 15.26 17.85 23.73 2329.99 3814 33.1 134.43 0.30 0.02 -2431.04 54 -2.43 0.00 47911c28 dev development
10 2025-11-11T04:35:43Z ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry 5760 1.845 3104.7 15.26 17.82 23.73 1586.43 3814 33.1 106.98 0.21 0.02 -2431.04 54 -2.43 000 47911c28 dev development
11 2025-11-11T04:37:04Z ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry 5760 2.115 2686.2 15.71 17.55 23.91 1762.82 3814 33.1 178.50 0.23 0.03 -2431.04 54 -2.43 0.00 567de2e5 dev development
12 2025-11-11T04:41:15Z ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry 5760 1.185 4835.7 15.26 18.02 23.47 1031.96 3814 33.1 73.13 0.13 0.01 -2431.04 54 -2.43 0.00 2a0fbf9b dev development
13 2025-11-11T04:43:07Z ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry 5760 1.515 3780.7 15.27 17.97 23.71 1356.28 3814 33.1 75.68 0.18 0.01 -2431.04 54 -2.43 0.00 2a0fbf9b dev development
14 2025-11-11T04:44:55Z ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry 5760 1.205 4763.7 15.27 22.29 23.54 1051.53 3828 33.2 73.41 0.14 0.01 -926.35 54 -0.93 0.00 2a0fbf9b dev development
15 2025-11-11T04:45:58Z ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry 5760 1.355 4225.9 15.25 22.63 23.51 1206.02 3828 33.2 73.26 0.16 0.01 -926.35 54 -0.93 0.00 2a0fbf9b dev development
16 2025-11-11T04:47:41Z ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry 5760 2.245 2561.0 15.26 22.13 23.55 1985.06 3828 33.2 123.63 0.26 0.02 -926.35 54 -0.93 0.00 2a0fbf9b dev development
17 2025-11-11T04:52:39Z ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry 5760 1.115 5152.6 15.26 13.71 24.64 963.17 3828 33.2 77.20 0.13 0.01 24560.79 38 24.56 6015 2a0fbf9b dev development
18 2025-11-11T04:53:16Z ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry 5760 1.455 3933.6 15.26 13.56 25.20 1240.30 3828 33.2 112.03 0.16 0.02 24560.79 38 24.56 6015 2a0fbf9b dev development
19 2025-11-11T04:56:15Z ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry 5760 2.14 2683.2 15.27 13.85 25.18 1763.50 3828 33.2 204.74 0.23 0.04 24560.79 38 24.56 6015 2a0fbf9b dev development
20 2025-11-11T04:57:14Z ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry 5760 1.99 2883.5 15.26 13.73 25.11 1589.82 3828 33.2 258.98 0.21 0.04 24560.79 38 24.56 6015 2a0fbf9b dev development
21 2025-11-11T04:59:09Z ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry 5760 2.695 2127.6 15.26 13.64 24.65 2283.69 3828 33.2 209.33 0.30 0.04 24560.79 38 24.56 6015 2a0fbf9b dev development
22 2025-11-11T05:13:30Z ExecuteBacktest_With_Large_Dataset_Should_Show_Performance_Telemetry 5760 2.49 2300.8 15.27 13.68 25.14 2085.01 3828 33.2 232.91 0.27 0.04 24560.79 38 24.56 6015 2a0fbf9b dev development