Fix positions for backtests
This commit is contained in:
@@ -177,9 +177,10 @@ public class BacktestExecutor
|
|||||||
// Pre-calculate indicator values once for all candles to optimize performance
|
// Pre-calculate indicator values once for all candles to optimize performance
|
||||||
// This avoids recalculating indicators for every candle iteration
|
// This avoids recalculating indicators for every candle iteration
|
||||||
Dictionary<IndicatorType, IndicatorsResultBase> preCalculatedIndicatorValues = null;
|
Dictionary<IndicatorType, IndicatorsResultBase> preCalculatedIndicatorValues = null;
|
||||||
var indicatorCalcStart = Stopwatch.GetTimestamp();
|
if (config.Scenario != null && false)
|
||||||
if (config.Scenario != null)
|
|
||||||
{
|
{
|
||||||
|
var indicatorCalcStart = Stopwatch.GetTimestamp();
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
_logger.LogInformation("⚡ Pre-calculating indicator values for {IndicatorCount} indicators",
|
_logger.LogInformation("⚡ Pre-calculating indicator values for {IndicatorCount} indicators",
|
||||||
@@ -222,15 +223,12 @@ public class BacktestExecutor
|
|||||||
var initialBalance = config.BotTradingBalance;
|
var initialBalance = config.BotTradingBalance;
|
||||||
|
|
||||||
// Pre-allocate and populate candle structures for maximum performance
|
// Pre-allocate and populate candle structures for maximum performance
|
||||||
var orderedCandles = candles.OrderBy(c => c.Date).ToList();
|
var orderedCandles = candles.ToList();
|
||||||
|
|
||||||
// Skip pre-calculated signals - the approach was flawed and caused performance regression
|
// Skip pre-calculated signals - the approach was flawed and caused performance regression
|
||||||
// The signal calculation depends on rolling window state and cannot be pre-calculated effectively
|
// The signal calculation depends on rolling window state and cannot be pre-calculated effectively
|
||||||
|
|
||||||
// Use optimized rolling window approach - TradingBox.GetSignal only needs last 600 candles
|
// Use optimized rolling window approach - TradingBox.GetSignal only needs last 600 candles
|
||||||
const int rollingWindowSize = 600;
|
|
||||||
var rollingCandles = new List<Candle>(rollingWindowSize); // Pre-allocate capacity for better performance
|
|
||||||
var fixedCandlesHashSet = new HashSet<Candle>(rollingWindowSize); // Reuse HashSet to avoid allocations
|
|
||||||
var candlesProcessed = 0;
|
var candlesProcessed = 0;
|
||||||
|
|
||||||
// Signal caching optimization - reduce signal update frequency for better performance
|
// Signal caching optimization - reduce signal update frequency for better performance
|
||||||
@@ -242,6 +240,8 @@ public class BacktestExecutor
|
|||||||
var lastWalletCheck = 0;
|
var lastWalletCheck = 0;
|
||||||
var lastWalletBalance = config.BotTradingBalance;
|
var lastWalletBalance = config.BotTradingBalance;
|
||||||
|
|
||||||
|
var fixedCandles = new HashSet<Candle>();
|
||||||
|
|
||||||
// Track memory usage during processing
|
// Track memory usage during processing
|
||||||
var peakMemory = initialMemory;
|
var peakMemory = initialMemory;
|
||||||
const int memoryCheckInterval = 100; // Check memory every N candles to reduce GC.GetTotalMemory overhead
|
const int memoryCheckInterval = 100; // Check memory every N candles to reduce GC.GetTotalMemory overhead
|
||||||
@@ -258,44 +258,15 @@ public class BacktestExecutor
|
|||||||
// Process all candles with optimized rolling window approach
|
// Process all candles with optimized rolling window approach
|
||||||
foreach (var candle in orderedCandles)
|
foreach (var candle in orderedCandles)
|
||||||
{
|
{
|
||||||
// Maintain rolling window efficiently using List
|
|
||||||
rollingCandles.Add(candle);
|
|
||||||
|
|
||||||
if (rollingCandles.Count > rollingWindowSize)
|
|
||||||
{
|
|
||||||
// Remove oldest candle from both structures
|
|
||||||
var removedCandle = rollingCandles[0];
|
|
||||||
rollingCandles.RemoveAt(0);
|
|
||||||
fixedCandlesHashSet.Remove(removedCandle);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add to HashSet for reuse
|
// Add to HashSet for reuse
|
||||||
fixedCandlesHashSet.Add(candle);
|
fixedCandles.Add(candle);
|
||||||
tradingBot.LastCandle = candle;
|
tradingBot.LastCandle = candle;
|
||||||
|
|
||||||
// Smart signal caching - reduce signal update frequency for performance
|
|
||||||
// RSI and similar indicators don't need updates every candle for 15-minute data
|
|
||||||
var shouldSkipSignalUpdate = ShouldSkipSignalUpdate(currentCandle, totalCandles);
|
|
||||||
|
|
||||||
if (!shouldSkipSignalUpdate)
|
|
||||||
{
|
|
||||||
// Smart signal caching - reduce signal update frequency for performance
|
|
||||||
// RSI and similar indicators don't need updates every candle for 15-minute data
|
|
||||||
var signalUpdateStart = Stopwatch.GetTimestamp();
|
|
||||||
await tradingBot.UpdateSignals(fixedCandlesHashSet);
|
|
||||||
signalUpdateTotalTime += Stopwatch.GetElapsedTime(signalUpdateStart);
|
|
||||||
telemetry.TotalSignalUpdates++;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
signalUpdateSkipCount++;
|
|
||||||
// Skip signal update - reuse previous signal state
|
|
||||||
// This saves ~1ms per skipped update and improves performance significantly
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run with optimized backtest path (minimize async calls)
|
// Run with optimized backtest path (minimize async calls)
|
||||||
var backtestStepStart = Stopwatch.GetTimestamp();
|
var backtestStepStart = Stopwatch.GetTimestamp();
|
||||||
await RunOptimizedBacktestStep(tradingBot);
|
await tradingBot.UpdateSignals(fixedCandles);
|
||||||
|
await tradingBot.Run();
|
||||||
|
|
||||||
backtestStepTotalTime += Stopwatch.GetElapsedTime(backtestStepStart);
|
backtestStepTotalTime += Stopwatch.GetElapsedTime(backtestStepStart);
|
||||||
|
|
||||||
telemetry.TotalBacktestSteps++;
|
telemetry.TotalBacktestSteps++;
|
||||||
@@ -343,11 +314,12 @@ public class BacktestExecutor
|
|||||||
// Track peak memory usage (reduced frequency to minimize GC overhead)
|
// Track peak memory usage (reduced frequency to minimize GC overhead)
|
||||||
if (currentCandle - lastMemoryCheck >= memoryCheckInterval)
|
if (currentCandle - lastMemoryCheck >= memoryCheckInterval)
|
||||||
{
|
{
|
||||||
var currentMemory = GC.GetTotalMemory(false);
|
var currentMemory = GC.GetTotalMemory(false);
|
||||||
if (currentMemory > peakMemory)
|
if (currentMemory > peakMemory)
|
||||||
{
|
{
|
||||||
peakMemory = currentMemory;
|
peakMemory = currentMemory;
|
||||||
}
|
}
|
||||||
|
|
||||||
lastMemoryCheck = currentCandle;
|
lastMemoryCheck = currentCandle;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -554,7 +526,6 @@ public class BacktestExecutor
|
|||||||
/// Pre-calculates all signals for the entire backtest period
|
/// Pre-calculates all signals for the entire backtest period
|
||||||
/// This eliminates repeated GetSignal() calls during the backtest loop
|
/// This eliminates repeated GetSignal() calls during the backtest loop
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Converts a Backtest to LightBacktest
|
/// Converts a Backtest to LightBacktest
|
||||||
/// </summary>
|
/// </summary>
|
||||||
@@ -609,7 +580,6 @@ public class BacktestExecutor
|
|||||||
private async Task RunOptimizedBacktestStep(TradingBotBase tradingBot)
|
private async Task RunOptimizedBacktestStep(TradingBotBase tradingBot)
|
||||||
{
|
{
|
||||||
// Use the standard Run method but ensure it's optimized for backtests
|
// Use the standard Run method but ensure it's optimized for backtests
|
||||||
await tradingBot.Run();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -263,7 +263,8 @@ public class TradingBotBase : ITradingBot
|
|||||||
await UpdateSignals(candles, null);
|
await UpdateSignals(candles, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task UpdateSignals(HashSet<Candle> candles, Dictionary<DateTime, LightSignal> preCalculatedSignals = null)
|
public async Task UpdateSignals(HashSet<Candle> candles,
|
||||||
|
Dictionary<DateTime, LightSignal> preCalculatedSignals = null)
|
||||||
{
|
{
|
||||||
// Skip indicator checking if flipping is disabled and there's an open position
|
// Skip indicator checking if flipping is disabled and there's an open position
|
||||||
// This prevents unnecessary indicator calculations when we can't act on signals anyway
|
// This prevents unnecessary indicator calculations when we can't act on signals anyway
|
||||||
@@ -283,27 +284,10 @@ public class TradingBotBase : ITradingBot
|
|||||||
|
|
||||||
if (Config.IsForBacktest)
|
if (Config.IsForBacktest)
|
||||||
{
|
{
|
||||||
LightSignal backtestSignal;
|
var backtestSignal = TradingBox.GetSignal(candles, Config.Scenario, Signals, Config.Scenario.LoopbackPeriod,
|
||||||
|
PreCalculatedIndicatorValues);
|
||||||
if (preCalculatedSignals != null && LastCandle != null && preCalculatedSignals.TryGetValue(LastCandle.Date, out backtestSignal))
|
if (backtestSignal == null) return;
|
||||||
{
|
await AddSignal(backtestSignal);
|
||||||
// Use pre-calculated signal - fast path
|
|
||||||
if (backtestSignal == null) return;
|
|
||||||
await AddSignal(backtestSignal);
|
|
||||||
}
|
|
||||||
else if (candles != null)
|
|
||||||
{
|
|
||||||
// Fallback to original calculation if no pre-calculated signals available
|
|
||||||
backtestSignal = TradingBox.GetSignal(candles, Config.Scenario, Signals, Config.Scenario.LoopbackPeriod,
|
|
||||||
PreCalculatedIndicatorValues);
|
|
||||||
if (backtestSignal == null) return;
|
|
||||||
await AddSignal(backtestSignal);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
// No candles provided - skip signal update
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@@ -601,7 +585,8 @@ public class TradingBotBase : ITradingBot
|
|||||||
$"Checking position history before marking as closed...");
|
$"Checking position history before marking as closed...");
|
||||||
|
|
||||||
// Verify in exchange history before assuming it's closed
|
// Verify in exchange history before assuming it's closed
|
||||||
var (existsInHistory, hadWeb3ProxyError) = await CheckPositionInExchangeHistory(positionForSignal);
|
var (existsInHistory, hadWeb3ProxyError) =
|
||||||
|
await CheckPositionInExchangeHistory(positionForSignal);
|
||||||
|
|
||||||
if (hadWeb3ProxyError)
|
if (hadWeb3ProxyError)
|
||||||
{
|
{
|
||||||
@@ -781,7 +766,8 @@ public class TradingBotBase : ITradingBot
|
|||||||
|
|
||||||
// Position might be canceled by the broker
|
// Position might be canceled by the broker
|
||||||
// Check if position exists in exchange history with PnL before canceling
|
// Check if position exists in exchange history with PnL before canceling
|
||||||
var (positionFoundInHistory, hadWeb3ProxyError) = await CheckPositionInExchangeHistory(positionForSignal);
|
var (positionFoundInHistory, hadWeb3ProxyError) =
|
||||||
|
await CheckPositionInExchangeHistory(positionForSignal);
|
||||||
|
|
||||||
if (hadWeb3ProxyError)
|
if (hadWeb3ProxyError)
|
||||||
{
|
{
|
||||||
@@ -3087,7 +3073,8 @@ public class TradingBotBase : ITradingBot
|
|||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
Logger.LogError(ex, "Error during position recovery for position {PositionId}", positionForSignal.Identifier);
|
Logger.LogError(ex, "Error during position recovery for position {PositionId}",
|
||||||
|
positionForSignal.Identifier);
|
||||||
await LogWarning($"Position recovery failed due to exception: {ex.Message}");
|
await LogWarning($"Position recovery failed due to exception: {ex.Message}");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -68,7 +68,6 @@ public class EmaCrossIndicator : EmaBaseIndicatorBase
|
|||||||
// Filter pre-calculated EMA values to match the candles we're processing
|
// Filter pre-calculated EMA values to match the candles we're processing
|
||||||
ema = preCalculatedValues.Ema
|
ema = preCalculatedValues.Ema
|
||||||
.Where(e => candles.Any(c => c.Date == e.Date))
|
.Where(e => candles.Any(c => c.Date == e.Date))
|
||||||
.OrderBy(e => e.Date)
|
|
||||||
.ToList();
|
.ToList();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -61,7 +61,8 @@ public static class TradingBox
|
|||||||
Dictionary<string, LightSignal> previousSignal, int? loopbackPeriod,
|
Dictionary<string, LightSignal> previousSignal, int? loopbackPeriod,
|
||||||
Dictionary<IndicatorType, IndicatorsResultBase> preCalculatedIndicatorValues)
|
Dictionary<IndicatorType, IndicatorsResultBase> preCalculatedIndicatorValues)
|
||||||
{
|
{
|
||||||
return GetSignal(newCandles, scenario, previousSignal, _defaultConfig, loopbackPeriod, preCalculatedIndicatorValues);
|
return GetSignal(newCandles, scenario, previousSignal, _defaultConfig, loopbackPeriod,
|
||||||
|
preCalculatedIndicatorValues);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static LightSignal GetSignal(HashSet<Candle> newCandles, LightScenario lightScenario,
|
public static LightSignal GetSignal(HashSet<Candle> newCandles, LightScenario lightScenario,
|
||||||
@@ -75,24 +76,11 @@ public static class TradingBox
|
|||||||
Dictionary<IndicatorType, IndicatorsResultBase> preCalculatedIndicatorValues)
|
Dictionary<IndicatorType, IndicatorsResultBase> preCalculatedIndicatorValues)
|
||||||
{
|
{
|
||||||
var signalOnCandles = new List<LightSignal>();
|
var signalOnCandles = new List<LightSignal>();
|
||||||
// Optimize list creation - avoid redundant allocations and multiple ordering
|
|
||||||
List<Candle> limitedCandles;
|
|
||||||
if (newCandles.Count <= 600)
|
|
||||||
{
|
|
||||||
// For small sets, just order once
|
|
||||||
limitedCandles = newCandles.OrderBy(c => c.Date).ToList();
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
// For large sets, use more efficient approach: sort then take last
|
|
||||||
var sorted = newCandles.OrderBy(c => c.Date).ToList();
|
|
||||||
limitedCandles = sorted.Skip(sorted.Count - 600).ToList();
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach (var indicator in lightScenario.Indicators)
|
foreach (var indicator in lightScenario.Indicators)
|
||||||
{
|
{
|
||||||
IIndicator indicatorInstance = indicator.ToInterface();
|
IIndicator indicatorInstance = indicator.ToInterface();
|
||||||
|
|
||||||
// Use pre-calculated indicator values if available (for backtest optimization)
|
// Use pre-calculated indicator values if available (for backtest optimization)
|
||||||
List<LightSignal> signals;
|
List<LightSignal> signals;
|
||||||
if (preCalculatedIndicatorValues != null && preCalculatedIndicatorValues.ContainsKey(indicator.Type))
|
if (preCalculatedIndicatorValues != null && preCalculatedIndicatorValues.ContainsKey(indicator.Type))
|
||||||
@@ -122,10 +110,11 @@ public static class TradingBox
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var limitedCandles = newCandles.ToList();
|
||||||
// Optimized: limitedCandles is already ordered, no need to re-order
|
// Optimized: limitedCandles is already ordered, no need to re-order
|
||||||
var loopback = loopbackPeriod.HasValue && loopbackPeriod > 1 ? loopbackPeriod.Value : 1;
|
var loopback = loopbackPeriod.HasValue && loopbackPeriod > 1 ? loopbackPeriod.Value : 1;
|
||||||
var candleLoopback = limitedCandles.Count > loopback
|
var candleLoopback = limitedCandles.Count > loopback
|
||||||
? limitedCandles.Skip(limitedCandles.Count - loopback).ToList()
|
? limitedCandles.Skip(limitedCandles.Count - loopback).ToList()
|
||||||
: limitedCandles;
|
: limitedCandles;
|
||||||
|
|
||||||
if (!candleLoopback.Any())
|
if (!candleLoopback.Any())
|
||||||
|
|||||||
Reference in New Issue
Block a user