Update backtest worker to fix stuck backtest

This commit is contained in:
2025-12-26 04:27:08 +07:00
parent 5548de6815
commit f60277d81d
2 changed files with 76 additions and 389 deletions

View File

@@ -24,7 +24,7 @@ public class BundleBacktestHealthCheckWorker : BackgroundService
private readonly TimeSpan _inactiveThreshold = TimeSpan.FromMinutes(2); // Check bundles inactive for 2+ minutes private readonly TimeSpan _inactiveThreshold = TimeSpan.FromMinutes(2); // Check bundles inactive for 2+ minutes
private readonly TimeSpan private readonly TimeSpan
_stuckThreshold = TimeSpan.FromHours(2); // Consider bundle stuck if no progress for 2 hours _stuckThreshold = TimeSpan.FromMinutes(10); // Consider bundle stuck if no progress for 10 minutes
private readonly IMessengerService _messengerService; private readonly IMessengerService _messengerService;
@@ -153,11 +153,29 @@ public class BundleBacktestHealthCheckWorker : BackgroundService
return (StuckCount: 0, MissingJobsCount: 1, HealthyCount: 0); return (StuckCount: 0, MissingJobsCount: 1, HealthyCount: 0);
} }
// Check 2: Stuck bundle - Running for too long without progress // Calculate job status counts (used in multiple checks below)
var completedJobs = jobs.Count(j => j.Status == JobStatus.Completed);
var failedJobs = jobs.Count(j => j.Status == JobStatus.Failed);
var runningJobs = jobs.Count(j => j.Status == JobStatus.Running);
// Check 2: Stuck bundle - Running but no jobs are actually running
if (bundle.Status == BundleBacktestRequestStatus.Running) if (bundle.Status == BundleBacktestRequestStatus.Running)
{ {
var timeSinceUpdate = DateTime.UtcNow - bundle.UpdatedAt; var timeSinceUpdate = DateTime.UtcNow - bundle.UpdatedAt;
// Check if bundle is running but no jobs are being processed
// This indicates the old BundleBacktestWorker created the bundle but never created jobs
if (runningJobs == 0 && completedJobs == 0 && timeSinceUpdate > TimeSpan.FromMinutes(5))
{
_logger.LogWarning(
"⚠️ Bundle {BundleRequestId} is Running but has no jobs running or completed after {Minutes} minutes. " +
"This indicates jobs were never created or never picked up.",
bundle.RequestId, timeSinceUpdate.TotalMinutes);
await HandleStuckBundleAsync(bundle, timeSinceUpdate, jobs, backtestRepository, jobRepository);
return (StuckCount: 1, MissingJobsCount: 0, HealthyCount: 0);
}
// Check if bundle hasn't made progress in a long time
if (timeSinceUpdate > _stuckThreshold) if (timeSinceUpdate > _stuckThreshold)
{ {
await HandleStuckBundleAsync(bundle, timeSinceUpdate, jobs, backtestRepository, jobRepository); await HandleStuckBundleAsync(bundle, timeSinceUpdate, jobs, backtestRepository, jobRepository);
@@ -186,8 +204,6 @@ public class BundleBacktestHealthCheckWorker : BackgroundService
} }
// Check 4: Bundle with all jobs completed but bundle status not updated // Check 4: Bundle with all jobs completed but bundle status not updated
var completedJobs = jobs.Count(j => j.Status == JobStatus.Completed);
var failedJobs = jobs.Count(j => j.Status == JobStatus.Failed);
var totalProcessedJobs = completedJobs + failedJobs; var totalProcessedJobs = completedJobs + failedJobs;
if (totalProcessedJobs == bundle.TotalBacktests && if (totalProcessedJobs == bundle.TotalBacktests &&

View File

@@ -1,10 +1,6 @@
using System.Text.Json; using Managing.Application.Abstractions.Repositories;
using Managing.Application.Abstractions.Services; using Managing.Application.Abstractions.Services;
using Managing.Domain.Backtests; using Managing.Domain.Backtests;
using Managing.Domain.Bots;
using Managing.Domain.MoneyManagements;
using Managing.Domain.Scenarios;
using Managing.Domain.Strategies;
using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using static Managing.Common.Enums; using static Managing.Common.Enums;
@@ -12,17 +8,17 @@ using static Managing.Common.Enums;
namespace Managing.Application.Workers; namespace Managing.Application.Workers;
/// <summary> /// <summary>
/// Worker for processing bundle backtest requests /// Worker for creating jobs for bundle backtest requests.
/// This worker creates all jobs upfront and lets BacktestComputeWorker handle execution.
/// BundleBacktestHealthCheckWorker monitors bundle health and handles stuck scenarios.
/// </summary> /// </summary>
public class BundleBacktestWorker : BaseWorker<BundleBacktestWorker> public class BundleBacktestWorker : BaseWorker<BundleBacktestWorker>
{ {
private readonly IServiceProvider _serviceProvider; private readonly IServiceProvider _serviceProvider;
private readonly IMessengerService _messengerService;
private static readonly WorkerType _workerType = WorkerType.BundleBacktest; private static readonly WorkerType _workerType = WorkerType.BundleBacktest;
public BundleBacktestWorker( public BundleBacktestWorker(
IServiceProvider serviceProvider, IServiceProvider serviceProvider,
IMessengerService messengerService,
ILogger<BundleBacktestWorker> logger) : base( ILogger<BundleBacktestWorker> logger) : base(
_workerType, _workerType,
logger, logger,
@@ -30,21 +26,17 @@ public class BundleBacktestWorker : BaseWorker<BundleBacktestWorker>
serviceProvider) serviceProvider)
{ {
_serviceProvider = serviceProvider; _serviceProvider = serviceProvider;
_messengerService = messengerService;
} }
protected override async Task Run(CancellationToken cancellationToken) protected override async Task Run(CancellationToken cancellationToken)
{ {
var maxDegreeOfParallelism = 3;
using var semaphore = new SemaphoreSlim(maxDegreeOfParallelism);
var processingTasks = new List<Task>();
try try
{ {
// Create a new service scope to get fresh instances of services with scoped DbContext
using var scope = _serviceProvider.CreateScope(); using var scope = _serviceProvider.CreateScope();
var backtester = scope.ServiceProvider.GetRequiredService<IBacktester>(); var backtester = scope.ServiceProvider.GetRequiredService<IBacktester>();
var jobRepository = scope.ServiceProvider.GetRequiredService<IJobRepository>();
// Get pending bundle backtest requests // Get pending bundle backtest requests (these need jobs created)
var pendingRequests = var pendingRequests =
await backtester.GetBundleBacktestRequestsByStatusAsync(BundleBacktestRequestStatus.Pending); await backtester.GetBundleBacktestRequestsByStatusAsync(BundleBacktestRequestStatus.Pending);
@@ -53,24 +45,20 @@ public class BundleBacktestWorker : BaseWorker<BundleBacktestWorker>
if (cancellationToken.IsCancellationRequested) if (cancellationToken.IsCancellationRequested)
break; break;
await semaphore.WaitAsync(cancellationToken); try
var task = Task.Run(async () =>
{ {
try await ProcessBundleRequest(bundleRequest, backtester, jobRepository);
{ }
await ProcessBundleRequest(bundleRequest, cancellationToken); catch (Exception ex)
} {
finally _logger.LogError(ex, "Error processing bundle request {RequestId}", bundleRequest.RequestId);
{ // Mark bundle as failed so it doesn't get stuck in Pending status
semaphore.Release(); bundleRequest.Status = BundleBacktestRequestStatus.Failed;
} bundleRequest.ErrorMessage = $"Failed to create jobs: {ex.Message}";
}, cancellationToken); bundleRequest.UpdatedAt = DateTime.UtcNow;
processingTasks.Add(task); await backtester.UpdateBundleBacktestRequestAsync(bundleRequest);
}
} }
await Task.WhenAll(processingTasks);
await RetryUnfinishedBacktestsInFailedBundles(backtester, cancellationToken);
} }
catch (Exception ex) catch (Exception ex)
{ {
@@ -79,367 +67,50 @@ public class BundleBacktestWorker : BaseWorker<BundleBacktestWorker>
} }
} }
private async Task ProcessBundleRequest(BundleBacktestRequest bundleRequest, CancellationToken cancellationToken) private async Task ProcessBundleRequest(
{
// Create a new service scope for this task to avoid DbContext concurrency issues
using var scope = _serviceProvider.CreateScope();
var backtester = scope.ServiceProvider.GetRequiredService<IBacktester>();
try
{
_logger.LogInformation("Starting to process bundle backtest request {RequestId}", bundleRequest.RequestId);
// Update status to running
bundleRequest.Status = BundleBacktestRequestStatus.Running;
await backtester.UpdateBundleBacktestRequestAsync(bundleRequest);
// Generate backtest requests from the new variant structure
var backtestRequests = await GenerateBacktestRequestsFromVariants(bundleRequest);
if (backtestRequests == null || !backtestRequests.Any())
{
throw new InvalidOperationException("Failed to generate backtest requests from variants");
}
// Process each backtest request
for (int i = 0; i < backtestRequests.Count; i++)
{
if (cancellationToken.IsCancellationRequested)
break;
try
{
var runBacktestRequest = backtestRequests[i];
// Update current backtest being processed
bundleRequest.CurrentBacktest = $"Backtest {i + 1} of {backtestRequests.Count}";
await backtester.UpdateBundleBacktestRequestAsync(bundleRequest);
// Run the backtest directly with the strongly-typed request
var backtestId = await RunSingleBacktest(backtester, runBacktestRequest, bundleRequest, i,
cancellationToken);
if (!string.IsNullOrEmpty(backtestId))
{
bundleRequest.Results.Add(backtestId);
}
// Update progress
bundleRequest.CompletedBacktests++;
await backtester.UpdateBundleBacktestRequestAsync(bundleRequest);
_logger.LogInformation("Completed backtest {Index} for bundle request {RequestId}",
i + 1, bundleRequest.RequestId);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error processing backtest {Index} for bundle request {RequestId}",
i + 1, bundleRequest.RequestId);
bundleRequest.FailedBacktests++;
SentrySdk.CaptureException(ex);
await backtester.UpdateBundleBacktestRequestAsync(bundleRequest);
}
}
// Update final status and send notifications
if (bundleRequest.FailedBacktests == 0)
{
bundleRequest.Status = BundleBacktestRequestStatus.Completed;
// Send Telegram message to the user's channelId
await NotifyUser(bundleRequest);
}
else if (bundleRequest.CompletedBacktests == 0)
{
bundleRequest.Status = BundleBacktestRequestStatus.Failed;
bundleRequest.ErrorMessage = "All backtests failed";
}
else
{
bundleRequest.Status = BundleBacktestRequestStatus.Completed;
bundleRequest.ErrorMessage = $"{bundleRequest.FailedBacktests} backtests failed";
// Send Telegram message to the user's channelId even with partial failures
await NotifyUser(bundleRequest);
}
bundleRequest.CompletedAt = DateTime.UtcNow;
bundleRequest.CurrentBacktest = null;
await backtester.UpdateBundleBacktestRequestAsync(bundleRequest);
_logger.LogInformation("Completed processing bundle backtest request {RequestId} with status {Status}",
bundleRequest.RequestId, bundleRequest.Status);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error processing bundle backtest request {RequestId}", bundleRequest.RequestId);
bundleRequest.Status = BundleBacktestRequestStatus.Failed;
bundleRequest.ErrorMessage = ex.Message;
bundleRequest.CompletedAt = DateTime.UtcNow;
await backtester.UpdateBundleBacktestRequestAsync(bundleRequest);
}
}
private async Task NotifyUser(BundleBacktestRequest bundleRequest)
{
if (bundleRequest.User?.TelegramChannel != null)
{
var message =
$"⚠️ Bundle backtest '{bundleRequest.Name}' (ID: {bundleRequest.RequestId}) completed with {bundleRequest.FailedBacktests} failed backtests.";
await _messengerService.SendMessage(message, bundleRequest.User.TelegramChannel);
}
}
// Change RunSingleBacktest to accept RunBacktestRequest directly
private async Task<string> RunSingleBacktest(IBacktester backtester, RunBacktestRequest runBacktestRequest,
BundleBacktestRequest bundleRequest, BundleBacktestRequest bundleRequest,
int index, CancellationToken cancellationToken) IBacktester backtester,
IJobRepository jobRepository)
{ {
if (runBacktestRequest == null || runBacktestRequest.Config == null) // Check if jobs already exist for this bundle
{ var existingJobs = (await jobRepository.GetByBundleRequestIdAsync(bundleRequest.RequestId)).ToList();
_logger.LogError("Invalid RunBacktestRequest in bundle (null config)");
return string.Empty;
}
// Map MoneyManagement if (existingJobs.Any())
MoneyManagement moneyManagement = null;
if (!string.IsNullOrEmpty(runBacktestRequest.Config.MoneyManagementName))
{ {
// In worker context, we cannot resolve by name (no user/db), so skip or set null _logger.LogInformation(
// Optionally, log a warning "Bundle request {RequestId} already has {JobCount} jobs. Skipping job creation.",
_logger.LogWarning("MoneyManagementName provided but cannot resolve in worker context: {Name}", bundleRequest.RequestId, existingJobs.Count);
(string)runBacktestRequest.Config.MoneyManagementName);
} // Ensure bundle is in Running status if jobs exist
else if (runBacktestRequest.Config.MoneyManagement != null) if (bundleRequest.Status == BundleBacktestRequestStatus.Pending)
{
var mmReq = runBacktestRequest.Config.MoneyManagement;
moneyManagement = new MoneyManagement
{ {
Name = mmReq.Name, bundleRequest.Status = BundleBacktestRequestStatus.Running;
Timeframe = mmReq.Timeframe, bundleRequest.TotalBacktests = existingJobs.Count;
StopLoss = mmReq.StopLoss, bundleRequest.UpdatedAt = DateTime.UtcNow;
TakeProfit = mmReq.TakeProfit, await backtester.UpdateBundleBacktestRequestAsync(bundleRequest);
Leverage = mmReq.Leverage _logger.LogInformation(
}; "Updated bundle request {RequestId} to Running status",
moneyManagement.FormatPercentage();
}
// Map Scenario
LightScenario scenario = null;
if (runBacktestRequest.Config.Scenario != null)
{
var sReq = runBacktestRequest.Config.Scenario;
scenario = new LightScenario(sReq.Name, sReq.LookbackPeriod)
{
Indicators = sReq.Indicators?.Select(i => new LightIndicator(i.Name, i.Type)
{
MinimumHistory = i.MinimumHistory,
Period = i.Period,
FastPeriods = i.FastPeriods,
SlowPeriods = i.SlowPeriods,
SignalPeriods = i.SignalPeriods,
Multiplier = i.Multiplier,
SmoothPeriods = i.SmoothPeriods,
StochPeriods = i.StochPeriods,
CyclePeriods = i.CyclePeriods
}).ToList() ?? new List<LightIndicator>()
};
}
// Map TradingBotConfig
var backtestConfig = new TradingBotConfig
{
AccountName = runBacktestRequest.Config.AccountName,
MoneyManagement = moneyManagement,
Ticker = runBacktestRequest.Config.Ticker,
ScenarioName = runBacktestRequest.Config.ScenarioName,
Scenario = scenario,
Timeframe = runBacktestRequest.Config.Timeframe,
IsForWatchingOnly = runBacktestRequest.Config.IsForWatchingOnly,
BotTradingBalance = runBacktestRequest.Config.BotTradingBalance,
TradingType = TradingType.BacktestFutures,
CooldownPeriod = runBacktestRequest.Config.CooldownPeriod ?? 1,
MaxLossStreak = runBacktestRequest.Config.MaxLossStreak,
MaxPositionTimeHours = runBacktestRequest.Config.MaxPositionTimeHours,
FlipOnlyWhenInProfit = runBacktestRequest.Config.FlipOnlyWhenInProfit,
FlipPosition = runBacktestRequest.Config.FlipPosition,
Name = $"{bundleRequest.Name} #{index + 1}",
CloseEarlyWhenProfitable = runBacktestRequest.Config.CloseEarlyWhenProfitable,
UseSynthApi = runBacktestRequest.Config.UseSynthApi,
UseForPositionSizing = runBacktestRequest.Config.UseForPositionSizing,
UseForSignalFiltering = runBacktestRequest.Config.UseForSignalFiltering,
UseForDynamicStopLoss = runBacktestRequest.Config.UseForDynamicStopLoss
};
// Run the backtest (no user context)
var result = await backtester.RunTradingBotBacktest(
backtestConfig,
runBacktestRequest.StartDate,
runBacktestRequest.EndDate,
bundleRequest.User, // No user context in worker
true,
runBacktestRequest.WithCandles,
bundleRequest.RequestId.ToString() // Use bundleRequestId as requestId for traceability
);
_logger.LogInformation("Processed backtest for bundle request {RequestId}", bundleRequest.RequestId);
// Assume the backtest is created and you have its ID (e.g., backtest.Id)
// Return the backtest ID
return result.Id;
}
private async Task RetryUnfinishedBacktestsInFailedBundles(IBacktester backtester,
CancellationToken cancellationToken)
{
var failedBundles = await backtester.GetBundleBacktestRequestsByStatusAsync(BundleBacktestRequestStatus.Failed);
foreach (var failedBundle in failedBundles)
{
if (cancellationToken.IsCancellationRequested)
break;
// Use Results property to determine which backtests need to be retried
var succeededIds = new HashSet<string>(failedBundle.Results ?? new List<string>());
// Generate backtest requests from the new variant structure
var originalRequests = await GenerateBacktestRequestsFromVariants(failedBundle);
if (originalRequests == null || !originalRequests.Any()) continue;
for (int i = failedBundle.CompletedBacktests; i < originalRequests.Count; i++)
{
var expectedId = /* logic to compute expected backtest id for this request */ string.Empty;
// If this backtest was not run or did not succeed, re-run it
if (!succeededIds.Contains(expectedId))
{
var backtestId = await RunSingleBacktest(backtester, originalRequests[i], failedBundle, i,
cancellationToken);
if (!string.IsNullOrEmpty(backtestId))
{
failedBundle.Results?.Add(backtestId);
failedBundle.CompletedBacktests++;
await backtester.UpdateBundleBacktestRequestAsync(failedBundle);
}
}
}
// If all backtests succeeded, update the bundle status
if (failedBundle.CompletedBacktests == originalRequests.Count)
{
failedBundle.Status = BundleBacktestRequestStatus.Completed;
failedBundle.ErrorMessage = null; // Clear any previous error
failedBundle.CompletedAt = DateTime.UtcNow;
await backtester.UpdateBundleBacktestRequestAsync(failedBundle);
// Notify user about successful retry
await NotifyUser(failedBundle);
}
else
{
_logger.LogWarning("Bundle {RequestId} still has unfinished backtests after retry",
failedBundle.RequestId);
}
}
}
/// <summary>
/// Generates individual backtest requests from variant configuration
/// </summary>
private async Task<List<RunBacktestRequest>> GenerateBacktestRequestsFromVariants(
BundleBacktestRequest bundleRequest)
{
try
{
// Deserialize the variant configurations
var universalConfig =
JsonSerializer.Deserialize<BundleBacktestUniversalConfig>(bundleRequest.UniversalConfigJson);
var dateTimeRanges = JsonSerializer.Deserialize<List<DateTimeRange>>(bundleRequest.DateTimeRangesJson);
var moneyManagementVariants =
JsonSerializer.Deserialize<List<MoneyManagementVariant>>(bundleRequest.MoneyManagementVariantsJson);
var tickerVariants = JsonSerializer.Deserialize<List<Ticker>>(bundleRequest.TickerVariantsJson);
if (universalConfig == null || dateTimeRanges == null || moneyManagementVariants == null ||
tickerVariants == null)
{
_logger.LogError("Failed to deserialize variant configurations for bundle request {RequestId}",
bundleRequest.RequestId); bundleRequest.RequestId);
return new List<RunBacktestRequest>();
} }
return;
// Get the first account for the user using AccountService
using var scope = _serviceProvider.CreateScope();
var accountService = scope.ServiceProvider.GetRequiredService<IAccountService>();
var accounts =
await accountService.GetAccountsByUserAsync(bundleRequest.User, hideSecrets: true, getBalance: false);
var firstAccount = accounts.FirstOrDefault();
if (firstAccount == null)
{
_logger.LogError("No accounts found for user {UserId} in bundle request {RequestId}",
bundleRequest.User.Id, bundleRequest.RequestId);
return new List<RunBacktestRequest>();
}
var backtestRequests = new List<RunBacktestRequest>();
foreach (var dateRange in dateTimeRanges)
{
foreach (var mmVariant in moneyManagementVariants)
{
foreach (var ticker in tickerVariants)
{
var config = new TradingBotConfigRequest
{
AccountName = firstAccount.Name,
Ticker = ticker,
Timeframe = universalConfig.Timeframe,
IsForWatchingOnly = universalConfig.IsForWatchingOnly,
BotTradingBalance = universalConfig.BotTradingBalance,
Name =
$"{universalConfig.BotName}_{ticker}_{dateRange.StartDate:yyyyMMdd}_{dateRange.EndDate:yyyyMMdd}",
FlipPosition = universalConfig.FlipPosition,
CooldownPeriod = universalConfig.CooldownPeriod,
MaxLossStreak = universalConfig.MaxLossStreak,
Scenario = universalConfig.Scenario,
ScenarioName = universalConfig.ScenarioName,
MoneyManagement = mmVariant.MoneyManagement,
MaxPositionTimeHours = universalConfig.MaxPositionTimeHours,
CloseEarlyWhenProfitable = universalConfig.CloseEarlyWhenProfitable,
FlipOnlyWhenInProfit = universalConfig.FlipOnlyWhenInProfit,
UseSynthApi = universalConfig.UseSynthApi,
UseForPositionSizing = universalConfig.UseForPositionSizing,
UseForSignalFiltering = universalConfig.UseForSignalFiltering,
UseForDynamicStopLoss = universalConfig.UseForDynamicStopLoss,
TradingType = universalConfig.TradingType
};
var backtestRequest = new RunBacktestRequest
{
Config = config,
StartDate = dateRange.StartDate,
EndDate = dateRange.EndDate,
Balance = universalConfig.BotTradingBalance,
WatchOnly = universalConfig.WatchOnly,
Save = universalConfig.Save,
WithCandles = false, // Bundle backtests never return candles
MoneyManagement = mmVariant.MoneyManagement != null
? new MoneyManagement
{
Name = mmVariant.MoneyManagement.Name,
Timeframe = mmVariant.MoneyManagement.Timeframe,
StopLoss = mmVariant.MoneyManagement.StopLoss,
TakeProfit = mmVariant.MoneyManagement.TakeProfit,
Leverage = mmVariant.MoneyManagement.Leverage
}
: null
};
backtestRequests.Add(backtestRequest);
}
}
}
return backtestRequests;
}
catch (Exception ex)
{
_logger.LogError(ex, "Error generating backtest requests from variants for bundle request {RequestId}",
bundleRequest.RequestId);
return new List<RunBacktestRequest>();
} }
_logger.LogInformation("Creating jobs for bundle request {RequestId}", bundleRequest.RequestId);
// Create all jobs for this bundle using the service method
await backtester.CreateJobsForBundleRequestAsync(bundleRequest);
// Get the created jobs to set TotalBacktests
var createdJobs = (await jobRepository.GetByBundleRequestIdAsync(bundleRequest.RequestId)).ToList();
// Update bundle status to Running and set TotalBacktests
bundleRequest.Status = BundleBacktestRequestStatus.Running;
bundleRequest.TotalBacktests = createdJobs.Count;
bundleRequest.UpdatedAt = DateTime.UtcNow;
await backtester.UpdateBundleBacktestRequestAsync(bundleRequest);
_logger.LogInformation(
"Successfully created {JobCount} jobs for bundle request {RequestId}. Bundle is now Running.",
createdJobs.Count, bundleRequest.RequestId);
} }
} }