Add benchmark for backtest on the test

This commit is contained in:
2025-11-11 11:23:30 +07:00
parent 2ca77bc2f9
commit 14d101b63e
8 changed files with 360 additions and 43 deletions

View File

@@ -21,7 +21,7 @@ public class BundleBacktestHealthCheckWorker : BackgroundService
private readonly IServiceScopeFactory _scopeFactory;
private readonly ILogger<BundleBacktestHealthCheckWorker> _logger;
private readonly TimeSpan _checkInterval = TimeSpan.FromMinutes(30);
private readonly TimeSpan _inactiveThreshold = TimeSpan.FromMinutes(30); // Check bundles inactive for 30+ minutes
private readonly TimeSpan _inactiveThreshold = TimeSpan.FromMinutes(2); // Check bundles inactive for 2+ minutes
private readonly TimeSpan _stuckThreshold = TimeSpan.FromHours(2); // Consider bundle stuck if no progress for 2 hours
private readonly IMessengerService _messengerService;
@@ -90,8 +90,8 @@ public class BundleBacktestHealthCheckWorker : BackgroundService
.ToList();
_logger.LogInformation(
"Found {TotalCount} bundles (from {PendingTotal} pending and {RunningTotal} running) that haven't been updated in >30 minutes",
allBundlesToCheck.Count, pendingBundles.Count(), runningBundles.Count());
"Found {TotalCount} bundles (from {PendingTotal} pending and {RunningTotal} running) that haven't been updated in >{InactiveMinutes} minutes",
allBundlesToCheck.Count, pendingBundles.Count(), runningBundles.Count(), _inactiveThreshold.TotalMinutes);
var stuckBundlesCount = 0;
var missingJobsCount = 0;
@@ -163,13 +163,13 @@ public class BundleBacktestHealthCheckWorker : BackgroundService
if (bundle.Status == BundleBacktestRequestStatus.Pending)
{
var timeSinceCreation = DateTime.UtcNow - bundle.CreatedAt;
// If bundle has been pending for more than the stuck threshold, check job statuses
if (timeSinceCreation > _stuckThreshold)
{
var allJobsPending = jobs.All(j => j.Status == JobStatus.Pending);
var hasFailedJobs = jobs.Any(j => j.Status == JobStatus.Failed);
if (allJobsPending || hasFailedJobs)
{
await HandleStalePendingBundleAsync(bundle, timeSinceCreation, jobs, backtestRepository, jobRepository);
@@ -178,6 +178,18 @@ public class BundleBacktestHealthCheckWorker : BackgroundService
}
}
// Check 4: Bundle with all jobs completed but bundle status not updated
var completedJobs = jobs.Count(j => j.Status == JobStatus.Completed);
var failedJobs = jobs.Count(j => j.Status == JobStatus.Failed);
var totalProcessedJobs = completedJobs + failedJobs;
if (totalProcessedJobs == bundle.TotalBacktests &&
(bundle.Status == BundleBacktestRequestStatus.Running || bundle.Status == BundleBacktestRequestStatus.Pending))
{
await HandleCompletedBundleAsync(bundle, completedJobs, failedJobs, backtestRepository);
return (StuckCount: 0, MissingJobsCount: 0, HealthyCount: 1);
}
return (StuckCount: 0, MissingJobsCount: 0, HealthyCount: 1);
}
@@ -471,6 +483,39 @@ public class BundleBacktestHealthCheckWorker : BackgroundService
bundle.RequestId, bundle.Status);
}
private async Task HandleCompletedBundleAsync(
BundleBacktestRequest bundle,
int completedJobs,
int failedJobs,
IBacktestRepository backtestRepository)
{
_logger.LogInformation(
"✅ Bundle {BundleRequestId} has all jobs finished ({Completed} completed, {Failed} failed) but bundle status was {OldStatus}. Updating to Completed.",
bundle.RequestId, completedJobs, failedJobs, bundle.Status);
// Update bundle status to Completed (or keep as Completed if it was already)
bundle.Status = failedJobs == 0 ? BundleBacktestRequestStatus.Completed : BundleBacktestRequestStatus.Completed;
bundle.CompletedBacktests = completedJobs;
bundle.FailedBacktests = failedJobs;
bundle.CompletedAt = DateTime.UtcNow;
bundle.UpdatedAt = DateTime.UtcNow;
if (failedJobs > 0)
{
bundle.ErrorMessage = $"{failedJobs} backtests failed";
}
else
{
bundle.ErrorMessage = null; // Clear any previous error message
}
await backtestRepository.UpdateBundleBacktestRequestAsync(bundle);
_logger.LogInformation(
"Successfully updated bundle {BundleRequestId} status to {Status} with {Completed}/{Total} backtests completed",
bundle.RequestId, bundle.Status, bundle.CompletedBacktests, bundle.TotalBacktests);
}
private async Task HandleStalePendingBundleAsync(
BundleBacktestRequest bundle,
TimeSpan timeSinceCreation,
@@ -484,12 +529,12 @@ public class BundleBacktestHealthCheckWorker : BackgroundService
bundle.RequestId, timeSinceCreation.TotalHours, jobs.Count, jobs.All(j => j.Status == JobStatus.Pending));
var hasFailedJobs = jobs.Any(j => j.Status == JobStatus.Failed);
if (hasFailedJobs)
{
// If all jobs failed, mark bundle as failed
var failedJobCount = jobs.Count(j => j.Status == JobStatus.Failed);
if (failedJobCount == bundle.TotalBacktests)
{
_logger.LogInformation(
@@ -511,7 +556,7 @@ public class BundleBacktestHealthCheckWorker : BackgroundService
// All jobs are pending - just update the timestamp to avoid repeated warnings
bundle.UpdatedAt = DateTime.UtcNow;
await backtestRepository.UpdateBundleBacktestRequestAsync(bundle);
_logger.LogInformation(
"Bundle {BundleRequestId} has {JobCount} pending jobs waiting to be processed. Updated timestamp.",
bundle.RequestId, jobs.Count);