Add test for executor

src/Managing.Application/Backtests/BacktestExecutor.cs

@@ -54,6 +54,7 @@ public class BacktestExecutor
     /// <param name="save">Whether to save the backtest result</param>
     /// <param name="withCandles">Whether to include candles in the result</param>
     /// <param name="requestId">The request ID to associate with this backtest</param>
+    /// <param name="bundleRequestId">Optional bundle request ID to update with backtest result</param>
     /// <param name="metadata">Additional metadata</param>
     /// <param name="progressCallback">Optional callback for progress updates (0-100)</param>
     /// <returns>The lightweight backtest result</returns>
@@ -64,6 +65,7 @@ public class BacktestExecutor
         bool save = false,
         bool withCandles = false,
         string requestId = null,
+        Guid? bundleRequestId = null,
         object metadata = null,
         Func<int, Task> progressCallback = null)
     {
@@ -72,6 +74,9 @@ public class BacktestExecutor
             throw new Exception("No candle to backtest");
         }
 
+        // Start performance tracking
+        var backtestStartTime = DateTime.UtcNow;
+
         // Ensure user has accounts loaded
         if (user.Accounts == null || !user.Accounts.Any())
         {
@@ -129,23 +134,64 @@ public class BacktestExecutor
         tradingBot.WalletBalances.Add(candles.FirstOrDefault()!.Date, config.BotTradingBalance);
         var initialBalance = config.BotTradingBalance;
 
-        var fixedCandles = new HashSet<Candle>();
-        var lastProgressUpdate = DateTime.UtcNow;
-        const int progressUpdateIntervalMs = 1000; // Update progress every second
+        // Pre-allocate and populate candle structures for maximum performance
+        var orderedCandles = candles.OrderBy(c => c.Date).ToList();
 
-        // Process all candles
-        foreach (var candle in candles)
+        // Use optimized rolling window approach - TradingBox.GetSignal only needs last 600 candles
+        const int rollingWindowSize = 600;
+        var rollingCandles = new LinkedList<Candle>();
+        var fixedCandles = new HashSet<Candle>(rollingWindowSize);
+        var candlesProcessed = 0;
+
+        var lastProgressUpdate = DateTime.UtcNow;
+        const int progressUpdateIntervalMs = 5000; // Update progress every 5 seconds to reduce database load
+        const int walletCheckInterval = 10; // Check wallet balance every N candles instead of every candle
+        var lastWalletCheck = 0;
+        var lastWalletBalance = config.BotTradingBalance;
+
+        // Process all candles with optimized rolling window approach
+        foreach (var candle in orderedCandles)
         {
+            // Maintain rolling window efficiently using LinkedList
+            rollingCandles.AddLast(candle);
             fixedCandles.Add(candle);
+
+            if (rollingCandles.Count > rollingWindowSize)
+            {
+                var removedCandle = rollingCandles.First!.Value;
+                rollingCandles.RemoveFirst();
+                fixedCandles.Remove(removedCandle);
+            }
+
             tradingBot.LastCandle = candle;
 
-            // Update signals manually only for backtesting
+            // Update signals and run trading logic with optimized rolling window
+            // For backtests, we can optimize by reducing async overhead
             await tradingBot.UpdateSignals(fixedCandles);
-            await tradingBot.Run();
+
+            // Run with optimized backtest path (minimize async calls)
+            await RunOptimizedBacktestStep(tradingBot);
 
             currentCandle++;
+            candlesProcessed++;
 
-            // Update progress callback if provided
+            // Optimized wallet balance check - only check every N candles and cache result
+            if (currentCandle - lastWalletCheck >= walletCheckInterval)
+            {
+                lastWalletBalance = tradingBot.WalletBalances.Values.LastOrDefault();
+                lastWalletCheck = currentCandle;
+
+                if (lastWalletBalance < Constants.GMX.Config.MinimumPositionAmount)
+                {
+                    _logger.LogWarning(
+                        "Backtest stopped early: Wallet balance fell below {MinimumPositionAmount} USDC (Current: {CurrentBalance:F2} USDC) at candle {CurrentCandle}/{TotalCandles} from {CandleDate}",
+                        Constants.GMX.Config.MinimumPositionAmount, lastWalletBalance, currentCandle, totalCandles,
+                        candle.Date.ToString("yyyy-MM-dd HH:mm"));
+                    break;
+                }
+            }
+
+            // Update progress callback if provided (optimized frequency)
             var currentPercentage = (currentCandle * 100) / totalCandles;
             var timeSinceLastUpdate = (DateTime.UtcNow - lastProgressUpdate).TotalMilliseconds;
             if (progressCallback != null && (timeSinceLastUpdate >= progressUpdateIntervalMs || currentPercentage >= lastLoggedPercentage + 10))
@@ -161,7 +207,7 @@ public class BacktestExecutor
                 lastProgressUpdate = DateTime.UtcNow;
             }
 
-            // Log progress every 10%
+            // Log progress every 10% (reduced frequency)
             if (currentPercentage >= lastLoggedPercentage + 10)
             {
                 lastLoggedPercentage = currentPercentage;
@@ -169,21 +215,20 @@ public class BacktestExecutor
                     "Backtest progress: {Percentage}% ({CurrentCandle}/{TotalCandles} candles processed)",
                     currentPercentage, currentCandle, totalCandles);
             }
-
-            // Check if wallet balance fell below 10 USDC and break if so
-            var currentWalletBalance = tradingBot.WalletBalances.Values.LastOrDefault();
-            if (currentWalletBalance < Constants.GMX.Config.MinimumPositionAmount)
-            {
-                _logger.LogWarning(
-                    "Backtest stopped early: Wallet balance fell below {MinimumPositionAmount} USDC (Current: {CurrentBalance:F2} USDC) at candle {CurrentCandle}/{TotalCandles} from {CandleDate}",
-                    Constants.GMX.Config.MinimumPositionAmount, currentWalletBalance, currentCandle, totalCandles,
-                    candle.Date.ToString("yyyy-MM-dd HH:mm"));
-                break;
-            }
         }
 
         _logger.LogInformation("Backtest processing completed. Calculating final results...");
 
+        // Log performance metrics
+        var backtestEndTime = DateTime.UtcNow;
+        var totalExecutionTime = backtestEndTime - backtestStartTime;
+        var candlesPerSecond = totalCandles / totalExecutionTime.TotalSeconds;
+
+        _logger.LogInformation(
+            "Backtest performance metrics: {TotalCandles} candles processed in {ExecutionTime:F2}s ({CandlesPerSecond:F1} candles/sec)",
+            totalCandles, totalExecutionTime.TotalSeconds, candlesPerSecond);
+
+        // Calculate final results (using existing optimized methods)
         var finalPnl = tradingBot.GetProfitAndLoss();
         var winRate = tradingBot.GetWinRate();
         var stats = TradingHelpers.GetStatistics(tradingBot.WalletBalances);
@@ -240,6 +285,12 @@ public class BacktestExecutor
         if (save && user != null)
         {
             await _backtestRepository.InsertBacktestForUserAsync(user, result);
+
+            // Update bundle request if provided
+            if (bundleRequestId.HasValue)
+            {
+                await UpdateBundleRequestWithBacktestResult(user, bundleRequestId.Value, result);
+            }
         }
 
         // Send notification if backtest meets criteria
@@ -297,6 +348,118 @@ public class BacktestExecutor
         return tradingBot;
     }
 
+    /// <summary>
+    /// Optimized backtest step execution - delegate to standard Run but with backtest optimizations
+    /// </summary>
+    private async Task RunOptimizedBacktestStep(TradingBotBase tradingBot)
+    {
+        // Use the standard Run method but ensure it's optimized for backtests
+        await tradingBot.Run();
+    }
+
+
+    /// <summary>
+    /// Updates bundle request with the completed backtest result
+    /// </summary>
+    private async Task UpdateBundleRequestWithBacktestResult(User user, Guid bundleRequestId, Backtest backtest)
+    {
+        try
+        {
+            using var scope = _scopeFactory.CreateScope();
+            var backtestRepository = scope.ServiceProvider.GetRequiredService<IBacktestRepository>();
+            var jobRepository = scope.ServiceProvider.GetRequiredService<IJobRepository>();
+            var webhookService = scope.ServiceProvider.GetRequiredService<IWebhookService>();
+
+            // Get bundle request
+            var bundleRequest = backtestRepository.GetBundleBacktestRequestByIdForUser(user, bundleRequestId);
+            if (bundleRequest == null)
+            {
+                _logger.LogWarning("Bundle request {BundleRequestId} not found for user {UserId}", bundleRequestId, user.Id);
+                return;
+            }
+
+            var previousStatus = bundleRequest.Status;
+
+            // Get all jobs for this bundle to calculate progress
+            var jobs = await jobRepository.GetByBundleRequestIdAsync(bundleRequestId);
+            var completedJobs = jobs.Count(j => j.Status == JobStatus.Completed);
+            var failedJobs = jobs.Count(j => j.Status == JobStatus.Failed);
+            var runningJobs = jobs.Count(j => j.Status == JobStatus.Running);
+            var totalJobs = jobs.Count();
+
+            // Update bundle request progress
+            bundleRequest.CompletedBacktests = completedJobs;
+            bundleRequest.FailedBacktests = failedJobs;
+
+            // Update status based on job states
+            if (completedJobs + failedJobs == totalJobs)
+            {
+                // All jobs completed or failed
+                if (failedJobs == 0)
+                {
+                    bundleRequest.Status = BundleBacktestRequestStatus.Completed;
+                }
+                else if (completedJobs == 0)
+                {
+                    bundleRequest.Status = BundleBacktestRequestStatus.Failed;
+                    bundleRequest.ErrorMessage = "All backtests failed";
+                }
+                else
+                {
+                    bundleRequest.Status = BundleBacktestRequestStatus.Completed;
+                    bundleRequest.ErrorMessage = $"{failedJobs} backtests failed";
+                }
+                bundleRequest.CompletedAt = DateTime.UtcNow;
+                bundleRequest.CurrentBacktest = null;
+            }
+            else if (runningJobs > 0)
+            {
+                // Some jobs still running
+                bundleRequest.Status = BundleBacktestRequestStatus.Running;
+            }
+
+            // Update results list with the new backtest ID
+            var resultsList = bundleRequest.Results?.ToList() ?? new List<string>();
+            if (!resultsList.Contains(backtest.Id))
+            {
+                resultsList.Add(backtest.Id);
+                bundleRequest.Results = resultsList;
+            }
+
+            await backtestRepository.UpdateBundleBacktestRequestAsync(bundleRequest);
+
+            // Send webhook notification if bundle request just completed
+            if (previousStatus != BundleBacktestRequestStatus.Completed &&
+                bundleRequest.Status == BundleBacktestRequestStatus.Completed &&
+                !string.IsNullOrEmpty(user.TelegramChannel))
+            {
+                var message = $"✅ Bundle backtest '{bundleRequest.Name}' (ID: {bundleRequest.RequestId}) completed successfully. " +
+                             $"Completed: {completedJobs}/{totalJobs} backtests" +
+                             (failedJobs > 0 ? $", Failed: {failedJobs}" : "") +
+                             $". Results: {resultsList.Count} backtest(s) generated.";
+
+                await webhookService.SendMessage(message, user.TelegramChannel);
+            }
+            else if (previousStatus != BundleBacktestRequestStatus.Failed &&
+                     bundleRequest.Status == BundleBacktestRequestStatus.Failed &&
+                     !string.IsNullOrEmpty(user.TelegramChannel))
+            {
+                var message = $"❌ Bundle backtest '{bundleRequest.Name}' (ID: {bundleRequest.RequestId}) failed. " +
+                             $"All {totalJobs} backtests failed. Error: {bundleRequest.ErrorMessage}";
+
+                await webhookService.SendMessage(message, user.TelegramChannel);
+            }
+
+            _logger.LogInformation(
+                "Updated bundle request {BundleRequestId} with backtest {BacktestId}: {Completed}/{Total} completed, {Failed} failed, {Running} running",
+                bundleRequestId, backtest.Id, completedJobs, totalJobs, failedJobs, runningJobs);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Failed to update bundle request {BundleRequestId} with backtest {BacktestId}", bundleRequestId, backtest.Id);
+        }
+    }
+
     /// <summary>
     /// Sends notification if backtest meets criteria
     /// </summary>

src/Managing.Application/Backtests/BacktestExecutorAdapter.cs

@@ -63,6 +63,7 @@ public class BacktestExecutorAdapter : IBacktester
             save,
             withCandles,
             requestId,
+            bundleRequestId: null,
             metadata,
             progressCallback: null);
 
@@ -85,6 +86,7 @@ public class BacktestExecutorAdapter : IBacktester
             save: false,
             withCandles,
             requestId,
+            bundleRequestId: null,
             metadata,
             progressCallback: null);
 

src/Managing.Application/GeneticService.cs

@@ -350,9 +350,10 @@ public class GeneticService : IGeneticService
 
             // Load candles once at the beginning to avoid repeated database queries
             // This significantly reduces database connections during genetic algorithm execution
-            _logger.LogInformation("Loading candles for genetic algorithm {RequestId}: {Ticker} on {Timeframe} from {StartDate} to {EndDate}",
+            _logger.LogInformation(
+                "Loading candles for genetic algorithm {RequestId}: {Ticker} on {Timeframe} from {StartDate} to {EndDate}",
                 request.RequestId, request.Ticker, request.Timeframe, request.StartDate, request.EndDate);
-            
+
             HashSet<Candle> candles;
             try
             {
@@ -366,13 +367,13 @@ public class GeneticService : IGeneticService
                         request.EndDate
                     )
                 );
-                
+
                 if (candles == null || candles.Count == 0)
                 {
                     throw new InvalidOperationException(
                         $"No candles found for {request.Ticker} on {request.Timeframe} from {request.StartDate} to {request.EndDate}");
                 }
-                
+
                 _logger.LogInformation("Loaded {CandleCount} candles for genetic algorithm {RequestId}",
                     candles.Count, request.RequestId);
             }
@@ -472,8 +473,9 @@ public class GeneticService : IGeneticService
                         {
                             // Reload the request from the database in the new scope
                             // Use the user from the original request to get the request by ID
-                            var dbRequest = geneticService.GetGeneticRequestByIdForUser(request.User, request.RequestId);
-                            
+                            var dbRequest =
+                                geneticService.GetGeneticRequestByIdForUser(request.User, request.RequestId);
+
                             if (dbRequest != null)
                             {
                                 // Update the loaded request with current generation data
@@ -482,13 +484,14 @@ public class GeneticService : IGeneticService
                                 dbRequest.BestChromosome = bestChromosomeJson;
                                 dbRequest.BestIndividual = bestIndividual;
                                 dbRequest.ProgressInfo = progressInfo;
-                                
+
                                 // Save the update
                                 await geneticService.UpdateGeneticRequestAsync(dbRequest);
                             }
                         });
 
-                    _logger.LogDebug("Updated genetic request {RequestId} at generation {Generation} with fitness {Fitness}",
+                    _logger.LogDebug(
+                        "Updated genetic request {RequestId} at generation {Generation} with fitness {Fitness}",
                         request.RequestId, generationCount, bestFitness);
 
                     // Check for cancellation
@@ -540,7 +543,7 @@ public class GeneticService : IGeneticService
             request.BestIndividual = bestChromosome?.ToString() ?? "unknown";
             request.CurrentGeneration = ga.GenerationsNumber;
             request.BestFitnessSoFar = bestFitness;
-            
+
             // Update BestChromosome if not already set
             if (bestChromosome != null && string.IsNullOrEmpty(request.BestChromosome))
             {
@@ -553,7 +556,7 @@ public class GeneticService : IGeneticService
                 }).ToArray();
                 request.BestChromosome = JsonSerializer.Serialize(geneValues);
             }
-            
+
             request.ProgressInfo = JsonSerializer.Serialize(new
             {
                 generation = ga.GenerationsNumber,
@@ -564,8 +567,9 @@ public class GeneticService : IGeneticService
             });
 
             await UpdateGeneticRequestAsync(request);
-            
-            _logger.LogInformation("Final update completed for genetic request {RequestId}. Generation: {Generation}, Best Fitness: {Fitness}",
+
+            _logger.LogInformation(
+                "Final update completed for genetic request {RequestId}. Generation: {Generation}, Best Fitness: {Fitness}",
                 request.RequestId, ga.GenerationsNumber, bestFitness);
 
             // Send notification about the completed genetic algorithm
@@ -893,7 +897,7 @@ public class TradingBotChromosome : ChromosomeBase
         return new TradingBotConfig
         {
             Name = $"Genetic_{request.RequestId}",
-            AccountName = "Oda-embedded",
+            AccountName = request.User.Accounts.FirstOrDefault().Name,
             Ticker = request.Ticker,
             Timeframe = request.Timeframe,
             BotTradingBalance = request.Balance,
@@ -1009,7 +1013,7 @@ public class TradingBotFitness : IFitness
     private static readonly SemaphoreSlim _dbSemaphore = new SemaphoreSlim(4, 4); // Limit concurrent DB operations
 
     public TradingBotFitness(
-        IServiceScopeFactory serviceScopeFactory, 
+        IServiceScopeFactory serviceScopeFactory,
         GeneticRequest request,
         HashSet<Candle> candles,
         ILogger<GeneticService> logger)
@@ -1051,7 +1055,9 @@ public class TradingBotFitness : IFitness
                         save: false, // Don't save backtest results for genetic algorithm
                         withCandles: false,
                         requestId: _request.RequestId,
-                        metadata: new GeneticBacktestMetadata(_geneticAlgorithm?.GenerationsNumber ?? 0, _request.RequestId)
+                        bundleRequestId: null, // Genetic algorithm doesn't use bundle requests
+                        metadata: new GeneticBacktestMetadata(_geneticAlgorithm?.GenerationsNumber ?? 0,
+                            _request.RequestId)
                     )
                 ).GetAwaiter().GetResult();
             }

src/Managing.Application/Users/UserService.cs

@@ -296,8 +296,7 @@ public class UserService : IUserService
 
     public async Task<User> GetUserByIdAsync(int userId)
     {
-        var allUsers = await _userRepository.GetAllUsersAsync();
-        var user = allUsers.FirstOrDefault(u => u.Id == userId);
+        var user = await _userRepository.GetUserByIdAsync(userId);
 
         if (user == null)
         {

src/Managing.Application/Workers/BacktestComputeWorker.cs

@@ -24,6 +24,7 @@ public class BacktestComputeWorker : BackgroundService
     private readonly BacktestComputeWorkerOptions _options;
     private readonly SemaphoreSlim _instanceSemaphore;
     private readonly ConcurrentDictionary<Guid, Task> _runningJobTasks = new();
+    private readonly ConcurrentDictionary<Guid, JobProgressTracker> _jobProgressTrackers = new();
     private readonly CancellationTokenSource _shutdownCts = new();
 
     public BacktestComputeWorker(
@@ -54,6 +55,9 @@ public class BacktestComputeWorker : BackgroundService
         // Background task for heartbeat updates
         var heartbeatTask = Task.Run(() => HeartbeatLoop(cancellationToken), cancellationToken);
 
+        // Background task for progress persistence
+        var progressPersistenceTask = Task.Run(() => ProgressPersistenceLoop(cancellationToken), cancellationToken);
+
         // Main job processing loop
         try
         {
@@ -230,30 +234,27 @@ public class BacktestComputeWorker : BackgroundService
                     $"No candles found for {config.Ticker} on {config.Timeframe} from {job.StartDate} to {job.EndDate}");
             }
 
-            // Progress callback to update job progress
-            Func<int, Task> progressCallback = async (percentage) =>
+            // Create progress tracker for this job
+            var progressTracker = new JobProgressTracker(job.Id, _logger);
+            _jobProgressTrackers.TryAdd(job.Id, progressTracker);
+
+            // Progress callback that only updates in-memory progress (non-blocking)
+            Func<int, Task> progressCallback = (percentage) =>
             {
-                try
+                // Check if job has been running too long
+                var elapsed = DateTime.UtcNow - jobStartTime;
+                if (elapsed.TotalMinutes > _options.JobTimeoutMinutes)
                 {
-                    // Check if job has been running too long
-                    var elapsed = DateTime.UtcNow - jobStartTime;
-                    if (elapsed.TotalMinutes > _options.JobTimeoutMinutes)
-                    {
-                        _logger.LogWarning(
-                            "Job {JobId} has been running for {ElapsedMinutes} minutes, exceeding timeout of {TimeoutMinutes} minutes",
-                            job.Id, elapsed.TotalMinutes, _options.JobTimeoutMinutes);
-                        throw new TimeoutException($"Job exceeded timeout of {_options.JobTimeoutMinutes} minutes");
-                    }
-                    
-                    job.ProgressPercentage = percentage;
-                    job.LastHeartbeat = DateTime.UtcNow;
-                    await jobRepository.UpdateAsync(job);
-                }
-                catch (Exception ex)
-                {
-                    _logger.LogWarning(ex, "Error updating job progress for job {JobId}", job.Id);
-                    throw; // Re-throw timeout exceptions
+                    _logger.LogWarning(
+                        "Job {JobId} has been running for {ElapsedMinutes} minutes, exceeding timeout of {TimeoutMinutes} minutes",
+                        job.Id, elapsed.TotalMinutes, _options.JobTimeoutMinutes);
+                    throw new TimeoutException($"Job exceeded timeout of {_options.JobTimeoutMinutes} minutes");
                 }
+
+                // Update progress in memory only - persistence happens in background
+                progressTracker.UpdateProgress(percentage);
+
+                return Task.CompletedTask; // Non-blocking
             };
 
             // Execute the backtest with timeout
@@ -270,6 +271,7 @@ public class BacktestComputeWorker : BackgroundService
                     save: true,
                     withCandles: false,
                     requestId: job.RequestId,
+                    bundleRequestId: job.BundleRequestId,
                     metadata: null,
                     progressCallback: progressCallback);
             }
@@ -293,6 +295,9 @@ public class BacktestComputeWorker : BackgroundService
 
             await jobRepository.UpdateAsync(job);
 
+            // Clean up progress tracker
+            _jobProgressTrackers.TryRemove(job.Id, out _);
+
             // Increment backtest count for the user's agent summary
             try
             {
@@ -310,11 +315,7 @@ public class BacktestComputeWorker : BackgroundService
                 "Completed backtest job {JobId}. Score: {Score}, PnL: {PnL}, Duration: {DurationMinutes:F1} minutes",
                 job.Id, result.Score, result.FinalPnl, elapsedTime.TotalMinutes);
 
-            // Update bundle request if this is part of a bundle
-            if (job.BundleRequestId.HasValue)
-            {
-                await UpdateBundleRequestProgress(job.BundleRequestId.Value, scope.ServiceProvider);
-            }
+            // Bundle request is now updated in the BacktestExecutor
         }
         catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
         {
@@ -326,6 +327,9 @@ public class BacktestComputeWorker : BackgroundService
             _logger.LogError(ex, "Error processing backtest job {JobId}", job.Id);
             SentrySdk.CaptureException(ex);
 
+            // Clean up progress tracker on failure
+            _jobProgressTrackers.TryRemove(job.Id, out _);
+
             await HandleJobFailure(job, ex, jobRepository, scope.ServiceProvider);
         }
     }
@@ -624,7 +628,7 @@ public class BacktestComputeWorker : BackgroundService
 
                 // Update heartbeat for all jobs assigned to this worker
                 var runningJobs = await jobRepository.GetRunningJobsByWorkerIdAsync(_options.WorkerId);
-                
+
                 foreach (var job in runningJobs)
                 {
                     job.LastHeartbeat = DateTime.UtcNow;
@@ -643,6 +647,62 @@ public class BacktestComputeWorker : BackgroundService
         }
     }
 
+    private async Task ProgressPersistenceLoop(CancellationToken cancellationToken)
+    {
+        while (!cancellationToken.IsCancellationRequested)
+        {
+            try
+            {
+                await Task.Delay(TimeSpan.FromSeconds(2), cancellationToken); // Check every 2 seconds
+
+                using var scope = _scopeFactory.CreateScope();
+                var jobRepository = scope.ServiceProvider.GetRequiredService<IJobRepository>();
+
+                // Process all progress trackers that need persistence
+                var trackersToPersist = _jobProgressTrackers
+                    .Where(kvp => kvp.Value.ShouldPersist())
+                    .ToList();
+
+                if (trackersToPersist.Count > 0)
+                {
+                    _logger.LogDebug("Persisting progress for {Count} jobs", trackersToPersist.Count);
+
+                    foreach (var (jobId, tracker) in trackersToPersist)
+                    {
+                        try
+                        {
+                            var (percentage, lastUpdate) = tracker.GetProgressForPersistence();
+
+                            // Get and update the job
+                            var job = await jobRepository.GetByIdAsync(jobId);
+                            if (job != null && job.Status == JobStatus.Running)
+                            {
+                                job.ProgressPercentage = percentage;
+                                job.LastHeartbeat = lastUpdate;
+                                await jobRepository.UpdateAsync(job);
+
+                                _logger.LogDebug("Persisted progress {Percentage}% for job {JobId}", percentage, jobId);
+                            }
+                        }
+                        catch (Exception ex)
+                        {
+                            _logger.LogWarning(ex, "Error persisting progress for job {JobId}", jobId);
+                        }
+                    }
+                }
+            }
+            catch (OperationCanceledException)
+            {
+                // Expected during shutdown, don't log as error
+                break;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Error in progress persistence loop");
+            }
+        }
+    }
+
     private async Task HandleJobFailure(
         Job job,
         Exception ex,
@@ -764,6 +824,73 @@ public class BacktestComputeWorker : BackgroundService
     }
 }
 
+/// <summary>
+/// Tracks job progress with batched database updates for performance optimization
+/// </summary>
+public class JobProgressTracker
+{
+    private readonly object _lock = new();
+    private int _lastPersistedPercentage;
+    private DateTime _lastPersistedTime;
+    private readonly ILogger _logger;
+
+    public Guid JobId { get; }
+    public int CurrentPercentage { get; private set; }
+    public DateTime LastUpdateTime { get; private set; }
+
+    public JobProgressTracker(Guid jobId, ILogger logger)
+    {
+        JobId = jobId;
+        _logger = logger;
+        _lastPersistedTime = DateTime.UtcNow;
+    }
+
+    /// <summary>
+    /// Updates progress in memory only - thread safe
+    /// </summary>
+    public void UpdateProgress(int percentage)
+    {
+        lock (_lock)
+        {
+            CurrentPercentage = percentage;
+            LastUpdateTime = DateTime.UtcNow;
+        }
+    }
+
+    /// <summary>
+    /// Checks if progress should be persisted to database based on time/percentage thresholds
+    /// </summary>
+    public bool ShouldPersist(int progressUpdateIntervalMs = 5000, int percentageThreshold = 5)
+    {
+        lock (_lock)
+        {
+            var timeSinceLastPersist = (DateTime.UtcNow - _lastPersistedTime).TotalMilliseconds;
+            var percentageSinceLastPersist = CurrentPercentage - _lastPersistedPercentage;
+
+            return timeSinceLastPersist >= progressUpdateIntervalMs ||
+                   percentageSinceLastPersist >= percentageThreshold ||
+                   CurrentPercentage >= 100; // Always persist completion
+        }
+    }
+
+    /// <summary>
+    /// Gets current progress and marks as persisted
+    /// </summary>
+    public (int percentage, DateTime lastUpdate) GetProgressForPersistence()
+    {
+        lock (_lock)
+        {
+            var percentage = CurrentPercentage;
+            var lastUpdate = LastUpdateTime;
+
+            _lastPersistedPercentage = percentage;
+            _lastPersistedTime = DateTime.UtcNow;
+
+            return (percentage, lastUpdate);
+        }
+    }
+}
+
 /// <summary>
 /// Configuration options for BacktestComputeWorker
 /// </summary>