Add benchmark for backtest on the test
This commit is contained in:
@@ -21,7 +21,7 @@ public class BundleBacktestHealthCheckWorker : BackgroundService
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
private readonly ILogger<BundleBacktestHealthCheckWorker> _logger;
|
||||
private readonly TimeSpan _checkInterval = TimeSpan.FromMinutes(30);
|
||||
private readonly TimeSpan _inactiveThreshold = TimeSpan.FromMinutes(30); // Check bundles inactive for 30+ minutes
|
||||
private readonly TimeSpan _inactiveThreshold = TimeSpan.FromMinutes(2); // Check bundles inactive for 2+ minutes
|
||||
private readonly TimeSpan _stuckThreshold = TimeSpan.FromHours(2); // Consider bundle stuck if no progress for 2 hours
|
||||
private readonly IMessengerService _messengerService;
|
||||
|
||||
@@ -90,8 +90,8 @@ public class BundleBacktestHealthCheckWorker : BackgroundService
|
||||
.ToList();
|
||||
|
||||
_logger.LogInformation(
|
||||
"Found {TotalCount} bundles (from {PendingTotal} pending and {RunningTotal} running) that haven't been updated in >30 minutes",
|
||||
allBundlesToCheck.Count, pendingBundles.Count(), runningBundles.Count());
|
||||
"Found {TotalCount} bundles (from {PendingTotal} pending and {RunningTotal} running) that haven't been updated in >{InactiveMinutes} minutes",
|
||||
allBundlesToCheck.Count, pendingBundles.Count(), runningBundles.Count(), _inactiveThreshold.TotalMinutes);
|
||||
|
||||
var stuckBundlesCount = 0;
|
||||
var missingJobsCount = 0;
|
||||
@@ -163,13 +163,13 @@ public class BundleBacktestHealthCheckWorker : BackgroundService
|
||||
if (bundle.Status == BundleBacktestRequestStatus.Pending)
|
||||
{
|
||||
var timeSinceCreation = DateTime.UtcNow - bundle.CreatedAt;
|
||||
|
||||
|
||||
// If bundle has been pending for more than the stuck threshold, check job statuses
|
||||
if (timeSinceCreation > _stuckThreshold)
|
||||
{
|
||||
var allJobsPending = jobs.All(j => j.Status == JobStatus.Pending);
|
||||
var hasFailedJobs = jobs.Any(j => j.Status == JobStatus.Failed);
|
||||
|
||||
|
||||
if (allJobsPending || hasFailedJobs)
|
||||
{
|
||||
await HandleStalePendingBundleAsync(bundle, timeSinceCreation, jobs, backtestRepository, jobRepository);
|
||||
@@ -178,6 +178,18 @@ public class BundleBacktestHealthCheckWorker : BackgroundService
|
||||
}
|
||||
}
|
||||
|
||||
// Check 4: Bundle with all jobs completed but bundle status not updated
|
||||
var completedJobs = jobs.Count(j => j.Status == JobStatus.Completed);
|
||||
var failedJobs = jobs.Count(j => j.Status == JobStatus.Failed);
|
||||
var totalProcessedJobs = completedJobs + failedJobs;
|
||||
|
||||
if (totalProcessedJobs == bundle.TotalBacktests &&
|
||||
(bundle.Status == BundleBacktestRequestStatus.Running || bundle.Status == BundleBacktestRequestStatus.Pending))
|
||||
{
|
||||
await HandleCompletedBundleAsync(bundle, completedJobs, failedJobs, backtestRepository);
|
||||
return (StuckCount: 0, MissingJobsCount: 0, HealthyCount: 1);
|
||||
}
|
||||
|
||||
return (StuckCount: 0, MissingJobsCount: 0, HealthyCount: 1);
|
||||
}
|
||||
|
||||
@@ -471,6 +483,39 @@ public class BundleBacktestHealthCheckWorker : BackgroundService
|
||||
bundle.RequestId, bundle.Status);
|
||||
}
|
||||
|
||||
private async Task HandleCompletedBundleAsync(
|
||||
BundleBacktestRequest bundle,
|
||||
int completedJobs,
|
||||
int failedJobs,
|
||||
IBacktestRepository backtestRepository)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"✅ Bundle {BundleRequestId} has all jobs finished ({Completed} completed, {Failed} failed) but bundle status was {OldStatus}. Updating to Completed.",
|
||||
bundle.RequestId, completedJobs, failedJobs, bundle.Status);
|
||||
|
||||
// Update bundle status to Completed (or keep as Completed if it was already)
|
||||
bundle.Status = failedJobs == 0 ? BundleBacktestRequestStatus.Completed : BundleBacktestRequestStatus.Completed;
|
||||
bundle.CompletedBacktests = completedJobs;
|
||||
bundle.FailedBacktests = failedJobs;
|
||||
bundle.CompletedAt = DateTime.UtcNow;
|
||||
bundle.UpdatedAt = DateTime.UtcNow;
|
||||
|
||||
if (failedJobs > 0)
|
||||
{
|
||||
bundle.ErrorMessage = $"{failedJobs} backtests failed";
|
||||
}
|
||||
else
|
||||
{
|
||||
bundle.ErrorMessage = null; // Clear any previous error message
|
||||
}
|
||||
|
||||
await backtestRepository.UpdateBundleBacktestRequestAsync(bundle);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Successfully updated bundle {BundleRequestId} status to {Status} with {Completed}/{Total} backtests completed",
|
||||
bundle.RequestId, bundle.Status, bundle.CompletedBacktests, bundle.TotalBacktests);
|
||||
}
|
||||
|
||||
private async Task HandleStalePendingBundleAsync(
|
||||
BundleBacktestRequest bundle,
|
||||
TimeSpan timeSinceCreation,
|
||||
@@ -484,12 +529,12 @@ public class BundleBacktestHealthCheckWorker : BackgroundService
|
||||
bundle.RequestId, timeSinceCreation.TotalHours, jobs.Count, jobs.All(j => j.Status == JobStatus.Pending));
|
||||
|
||||
var hasFailedJobs = jobs.Any(j => j.Status == JobStatus.Failed);
|
||||
|
||||
|
||||
if (hasFailedJobs)
|
||||
{
|
||||
// If all jobs failed, mark bundle as failed
|
||||
var failedJobCount = jobs.Count(j => j.Status == JobStatus.Failed);
|
||||
|
||||
|
||||
if (failedJobCount == bundle.TotalBacktests)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
@@ -511,7 +556,7 @@ public class BundleBacktestHealthCheckWorker : BackgroundService
|
||||
// All jobs are pending - just update the timestamp to avoid repeated warnings
|
||||
bundle.UpdatedAt = DateTime.UtcNow;
|
||||
await backtestRepository.UpdateBundleBacktestRequestAsync(bundle);
|
||||
|
||||
|
||||
_logger.LogInformation(
|
||||
"Bundle {BundleRequestId} has {JobCount} pending jobs waiting to be processed. Updated timestamp.",
|
||||
bundle.RequestId, jobs.Count);
|
||||
|
||||
Reference in New Issue
Block a user