Add genetic backtest to worker

This commit is contained in:
2025-11-09 03:32:08 +07:00
parent 7dba29c66f
commit 7e08e63dd1
30 changed files with 5056 additions and 232 deletions

View File

@@ -816,7 +816,7 @@ public class BacktestController : BaseController
// Get all jobs for this bundle // Get all jobs for this bundle
using var serviceScope = _serviceScopeFactory.CreateScope(); using var serviceScope = _serviceScopeFactory.CreateScope();
var jobRepository = serviceScope.ServiceProvider.GetRequiredService<IBacktestJobRepository>(); var jobRepository = serviceScope.ServiceProvider.GetRequiredService<IJobRepository>();
var jobs = await jobRepository.GetByBundleRequestIdAsync(bundleGuid); var jobs = await jobRepository.GetByBundleRequestIdAsync(bundleGuid);
var response = new BundleBacktestStatusResponse var response = new BundleBacktestStatusResponse
@@ -824,10 +824,10 @@ public class BacktestController : BaseController
BundleRequestId = bundleGuid, BundleRequestId = bundleGuid,
Status = bundleRequest.Status.ToString(), Status = bundleRequest.Status.ToString(),
TotalJobs = jobs.Count(), TotalJobs = jobs.Count(),
CompletedJobs = jobs.Count(j => j.Status == BacktestJobStatus.Completed), CompletedJobs = jobs.Count(j => j.Status == JobStatus.Completed),
FailedJobs = jobs.Count(j => j.Status == BacktestJobStatus.Failed), FailedJobs = jobs.Count(j => j.Status == JobStatus.Failed),
RunningJobs = jobs.Count(j => j.Status == BacktestJobStatus.Running), RunningJobs = jobs.Count(j => j.Status == JobStatus.Running),
PendingJobs = jobs.Count(j => j.Status == BacktestJobStatus.Pending), PendingJobs = jobs.Count(j => j.Status == JobStatus.Pending),
ProgressPercentage = bundleRequest.ProgressPercentage, ProgressPercentage = bundleRequest.ProgressPercentage,
CreatedAt = bundleRequest.CreatedAt, CreatedAt = bundleRequest.CreatedAt,
CompletedAt = bundleRequest.CompletedAt, CompletedAt = bundleRequest.CompletedAt,

View File

@@ -68,7 +68,7 @@ public class JobController : BaseController
/// <param name="jobId">The job ID to query</param> /// <param name="jobId">The job ID to query</param>
/// <returns>The job status and result if completed</returns> /// <returns>The job status and result if completed</returns>
[HttpGet("{jobId}")] [HttpGet("{jobId}")]
public async Task<ActionResult<BacktestJobStatusResponse>> GetJobStatus(string jobId) public async Task<ActionResult<JobStatusResponse>> GetJobStatus(string jobId)
{ {
if (!await IsUserAdmin()) if (!await IsUserAdmin())
{ {
@@ -82,7 +82,7 @@ public class JobController : BaseController
} }
using var serviceScope = _serviceScopeFactory.CreateScope(); using var serviceScope = _serviceScopeFactory.CreateScope();
var jobRepository = serviceScope.ServiceProvider.GetRequiredService<IBacktestJobRepository>(); var jobRepository = serviceScope.ServiceProvider.GetRequiredService<IJobRepository>();
var job = await jobRepository.GetByIdAsync(jobGuid); var job = await jobRepository.GetByIdAsync(jobGuid);
if (job == null) if (job == null)
@@ -90,7 +90,7 @@ public class JobController : BaseController
return NotFound($"Job with ID {jobId} not found."); return NotFound($"Job with ID {jobId} not found.");
} }
var response = new BacktestJobStatusResponse var response = new JobStatusResponse
{ {
JobId = job.Id, JobId = job.Id,
Status = job.Status.ToString(), Status = job.Status.ToString(),
@@ -99,7 +99,7 @@ public class JobController : BaseController
StartedAt = job.StartedAt, StartedAt = job.StartedAt,
CompletedAt = job.CompletedAt, CompletedAt = job.CompletedAt,
ErrorMessage = job.ErrorMessage, ErrorMessage = job.ErrorMessage,
Result = job.Status == BacktestJobStatus.Completed && !string.IsNullOrEmpty(job.ResultJson) Result = job.Status == JobStatus.Completed && !string.IsNullOrEmpty(job.ResultJson)
? JsonSerializer.Deserialize<LightBacktest>(job.ResultJson) ? JsonSerializer.Deserialize<LightBacktest>(job.ResultJson)
: null : null
}; };
@@ -156,16 +156,16 @@ public class JobController : BaseController
} }
// Parse status filter // Parse status filter
BacktestJobStatus? statusFilter = null; JobStatus? statusFilter = null;
if (!string.IsNullOrEmpty(status)) if (!string.IsNullOrEmpty(status))
{ {
if (Enum.TryParse<BacktestJobStatus>(status, true, out var parsedStatus)) if (Enum.TryParse<JobStatus>(status, true, out var parsedStatus))
{ {
statusFilter = parsedStatus; statusFilter = parsedStatus;
} }
else else
{ {
return BadRequest($"Invalid status value. Valid values are: {string.Join(", ", Enum.GetNames<BacktestJobStatus>())}"); return BadRequest($"Invalid status value. Valid values are: {string.Join(", ", Enum.GetNames<JobStatus>())}");
} }
} }
@@ -195,7 +195,7 @@ public class JobController : BaseController
} }
using var serviceScope = _serviceScopeFactory.CreateScope(); using var serviceScope = _serviceScopeFactory.CreateScope();
var jobRepository = serviceScope.ServiceProvider.GetRequiredService<IBacktestJobRepository>(); var jobRepository = serviceScope.ServiceProvider.GetRequiredService<IJobRepository>();
var (jobs, totalCount) = await jobRepository.GetPaginatedAsync( var (jobs, totalCount) = await jobRepository.GetPaginatedAsync(
page, page,
@@ -257,7 +257,7 @@ public class JobController : BaseController
} }
using var serviceScope = _serviceScopeFactory.CreateScope(); using var serviceScope = _serviceScopeFactory.CreateScope();
var jobRepository = serviceScope.ServiceProvider.GetRequiredService<IBacktestJobRepository>(); var jobRepository = serviceScope.ServiceProvider.GetRequiredService<IJobRepository>();
var summary = await jobRepository.GetSummaryAsync(); var summary = await jobRepository.GetSummaryAsync();

View File

@@ -3,9 +3,9 @@ using Managing.Domain.Backtests;
namespace Managing.Api.Models.Responses; namespace Managing.Api.Models.Responses;
/// <summary> /// <summary>
/// Response model for backtest job status /// Response model for job status
/// </summary> /// </summary>
public class BacktestJobStatusResponse public class JobStatusResponse
{ {
public Guid JobId { get; set; } public Guid JobId { get; set; }
public string Status { get; set; } = string.Empty; public string Status { get; set; } = string.Empty;

View File

@@ -4,14 +4,14 @@ using static Managing.Common.Enums;
namespace Managing.Application.Abstractions.Repositories; namespace Managing.Application.Abstractions.Repositories;
/// <summary> /// <summary>
/// Repository interface for managing backtest jobs in the queue system /// Repository interface for managing jobs in the queue system
/// </summary> /// </summary>
public interface IBacktestJobRepository public interface IJobRepository
{ {
/// <summary> /// <summary>
/// Creates a new backtest job /// Creates a new job
/// </summary> /// </summary>
Task<BacktestJob> CreateAsync(BacktestJob job); Task<Job> CreateAsync(Job job);
/// <summary> /// <summary>
/// Claims the next available job using PostgreSQL advisory locks. /// Claims the next available job using PostgreSQL advisory locks.
@@ -19,33 +19,33 @@ public interface IBacktestJobRepository
/// </summary> /// </summary>
/// <param name="workerId">The ID of the worker claiming the job</param> /// <param name="workerId">The ID of the worker claiming the job</param>
/// <param name="jobType">Optional job type filter. If null, claims any job type.</param> /// <param name="jobType">Optional job type filter. If null, claims any job type.</param>
Task<BacktestJob?> ClaimNextJobAsync(string workerId, JobType? jobType = null); Task<Job?> ClaimNextJobAsync(string workerId, JobType? jobType = null);
/// <summary> /// <summary>
/// Updates an existing job /// Updates an existing job
/// </summary> /// </summary>
Task UpdateAsync(BacktestJob job); Task UpdateAsync(Job job);
/// <summary> /// <summary>
/// Gets all jobs for a specific bundle request /// Gets all jobs for a specific bundle request
/// </summary> /// </summary>
Task<IEnumerable<BacktestJob>> GetByBundleRequestIdAsync(Guid bundleRequestId); Task<IEnumerable<Job>> GetByBundleRequestIdAsync(Guid bundleRequestId);
/// <summary> /// <summary>
/// Gets all jobs for a specific user /// Gets all jobs for a specific user
/// </summary> /// </summary>
Task<IEnumerable<BacktestJob>> GetByUserIdAsync(int userId); Task<IEnumerable<Job>> GetByUserIdAsync(int userId);
/// <summary> /// <summary>
/// Gets a job by its ID /// Gets a job by its ID
/// </summary> /// </summary>
Task<BacktestJob?> GetByIdAsync(Guid jobId); Task<Job?> GetByIdAsync(Guid jobId);
/// <summary> /// <summary>
/// Gets stale jobs (jobs that are Running but haven't sent a heartbeat in the specified timeout) /// Gets stale jobs (jobs that are Running but haven't sent a heartbeat in the specified timeout)
/// </summary> /// </summary>
/// <param name="timeoutMinutes">Number of minutes since last heartbeat to consider stale</param> /// <param name="timeoutMinutes">Number of minutes since last heartbeat to consider stale</param>
Task<IEnumerable<BacktestJob>> GetStaleJobsAsync(int timeoutMinutes = 5); Task<IEnumerable<Job>> GetStaleJobsAsync(int timeoutMinutes = 5);
/// <summary> /// <summary>
/// Resets stale jobs back to Pending status /// Resets stale jobs back to Pending status
@@ -55,12 +55,12 @@ public interface IBacktestJobRepository
/// <summary> /// <summary>
/// Gets all running jobs assigned to a specific worker /// Gets all running jobs assigned to a specific worker
/// </summary> /// </summary>
Task<IEnumerable<BacktestJob>> GetRunningJobsByWorkerIdAsync(string workerId); Task<IEnumerable<Job>> GetRunningJobsByWorkerIdAsync(string workerId);
/// <summary> /// <summary>
/// Gets all jobs for a specific genetic request ID /// Gets all jobs for a specific genetic request ID
/// </summary> /// </summary>
Task<IEnumerable<BacktestJob>> GetByGeneticRequestIdAsync(string geneticRequestId); Task<IEnumerable<Job>> GetByGeneticRequestIdAsync(string geneticRequestId);
/// <summary> /// <summary>
/// Gets paginated jobs with optional filters and sorting /// Gets paginated jobs with optional filters and sorting
@@ -75,12 +75,12 @@ public interface IBacktestJobRepository
/// <param name="workerId">Optional worker ID filter</param> /// <param name="workerId">Optional worker ID filter</param>
/// <param name="bundleRequestId">Optional bundle request ID filter</param> /// <param name="bundleRequestId">Optional bundle request ID filter</param>
/// <returns>Tuple of jobs and total count</returns> /// <returns>Tuple of jobs and total count</returns>
Task<(IEnumerable<BacktestJob> Jobs, int TotalCount)> GetPaginatedAsync( Task<(IEnumerable<Job> Jobs, int TotalCount)> GetPaginatedAsync(
int page, int page,
int pageSize, int pageSize,
string sortBy = "CreatedAt", string sortBy = "CreatedAt",
string sortOrder = "desc", string sortOrder = "desc",
BacktestJobStatus? status = null, JobStatus? status = null,
JobType? jobType = null, JobType? jobType = null,
int? userId = null, int? userId = null,
string? workerId = null, string? workerId = null,
@@ -109,7 +109,7 @@ public class JobSummary
/// </summary> /// </summary>
public class JobStatusCount public class JobStatusCount
{ {
public BacktestJobStatus Status { get; set; } public JobStatus Status { get; set; }
public int Count { get; set; } public int Count { get; set; }
} }
@@ -127,7 +127,7 @@ public class JobTypeCount
/// </summary> /// </summary>
public class JobStatusTypeCount public class JobStatusTypeCount
{ {
public BacktestJobStatus Status { get; set; } public JobStatus Status { get; set; }
public JobType JobType { get; set; } public JobType JobType { get; set; }
public int Count { get; set; } public int Count { get; set; }
} }

View File

@@ -43,12 +43,12 @@ namespace Managing.Application.Tests
var tradingBotLogger = TradingBaseTests.CreateTradingBotLogger(); var tradingBotLogger = TradingBaseTests.CreateTradingBotLogger();
var backtestLogger = TradingBaseTests.CreateBacktesterLogger(); var backtestLogger = TradingBaseTests.CreateBacktesterLogger();
ILoggerFactory loggerFactory = new NullLoggerFactory(); ILoggerFactory loggerFactory = new NullLoggerFactory();
var backtestJobLogger = loggerFactory.CreateLogger<BacktestJobService>(); var backtestJobLogger = loggerFactory.CreateLogger<JobService>();
var botService = new Mock<IBotService>().Object; var botService = new Mock<IBotService>().Object;
var agentService = new Mock<IAgentService>().Object; var agentService = new Mock<IAgentService>().Object;
var _scopeFactory = new Mock<IServiceScopeFactory>(); var _scopeFactory = new Mock<IServiceScopeFactory>();
var backtestJobRepository = new Mock<IBacktestJobRepository>().Object; var backtestJobRepository = new Mock<IJobRepository>().Object;
var backtestJobService = new BacktestJobService(backtestJobRepository, backtestRepository, kaigenService, backtestJobLogger); var backtestJobService = new JobService(backtestJobRepository, backtestRepository, kaigenService, backtestJobLogger);
_backtester = new Backtester(_exchangeService, backtestRepository, backtestLogger, _backtester = new Backtester(_exchangeService, backtestRepository, backtestLogger,
scenarioService, _accountService.Object, messengerService, kaigenService, hubContext, _scopeFactory.Object, scenarioService, _accountService.Object, messengerService, kaigenService, hubContext, _scopeFactory.Object,
backtestJobService); backtestJobService);

View File

@@ -0,0 +1,122 @@
using Managing.Application.Abstractions.Services;
using Managing.Application.Abstractions.Shared;
using Managing.Domain.Backtests;
using Managing.Domain.Bots;
using Managing.Domain.Candles;
using Managing.Domain.Users;
using Microsoft.Extensions.Logging;
using static Managing.Common.Enums;
namespace Managing.Application.Backtests;
/// <summary>
/// Adapter that wraps BacktestExecutor to implement IBacktester interface.
/// Used in compute workers where Backtester (with SignalR dependencies) is not available.
/// Only implements methods needed for genetic algorithm execution.
/// </summary>
public class BacktestExecutorAdapter : IBacktester
{
private readonly BacktestExecutor _executor;
private readonly IExchangeService _exchangeService;
private readonly ILogger<BacktestExecutorAdapter> _logger;
public BacktestExecutorAdapter(
BacktestExecutor executor,
IExchangeService exchangeService,
ILogger<BacktestExecutorAdapter> logger)
{
_executor = executor;
_exchangeService = exchangeService;
_logger = logger;
}
public async Task<LightBacktest> RunTradingBotBacktest(
TradingBotConfig config,
DateTime startDate,
DateTime endDate,
User user = null,
bool save = false,
bool withCandles = false,
string requestId = null,
object metadata = null)
{
// Load candles using ExchangeService
var candles = await _exchangeService.GetCandlesInflux(
TradingExchanges.Evm,
config.Ticker,
startDate,
config.Timeframe,
endDate);
if (candles == null || candles.Count == 0)
{
throw new InvalidOperationException(
$"No candles found for {config.Ticker} on {config.Timeframe} from {startDate} to {endDate}");
}
// Execute using BacktestExecutor
var result = await _executor.ExecuteAsync(
config,
candles,
user,
save,
withCandles,
requestId,
metadata,
progressCallback: null);
return result;
}
public async Task<LightBacktest> RunTradingBotBacktest(
TradingBotConfig config,
HashSet<Candle> candles,
User user = null,
bool withCandles = false,
string requestId = null,
object metadata = null)
{
// Execute using BacktestExecutor
var result = await _executor.ExecuteAsync(
config,
candles,
user,
save: false,
withCandles,
requestId,
metadata,
progressCallback: null);
return result;
}
// Methods not needed for compute worker - throw NotImplementedException
public Task<bool> DeleteBacktestAsync(string id) => throw new NotImplementedException("Not available in compute worker");
public bool DeleteBacktests() => throw new NotImplementedException("Not available in compute worker");
public IEnumerable<Backtest> GetBacktestsByUser(User user) => throw new NotImplementedException("Not available in compute worker");
public Task<IEnumerable<Backtest>> GetBacktestsByUserAsync(User user) => throw new NotImplementedException("Not available in compute worker");
public IEnumerable<Backtest> GetBacktestsByRequestId(Guid requestId) => throw new NotImplementedException("Not available in compute worker");
public Task<IEnumerable<Backtest>> GetBacktestsByRequestIdAsync(Guid requestId) => throw new NotImplementedException("Not available in compute worker");
public (IEnumerable<LightBacktest> Backtests, int TotalCount) GetBacktestsByRequestIdPaginated(Guid requestId, int page, int pageSize, string sortBy = "score", string sortOrder = "desc") => throw new NotImplementedException("Not available in compute worker");
public Task<(IEnumerable<LightBacktest> Backtests, int TotalCount)> GetBacktestsByRequestIdPaginatedAsync(Guid requestId, int page, int pageSize, string sortBy = "score", string sortOrder = "desc") => throw new NotImplementedException("Not available in compute worker");
public Task<Backtest> GetBacktestByIdForUserAsync(User user, string id) => throw new NotImplementedException("Not available in compute worker");
public Task<bool> DeleteBacktestByUserAsync(User user, string id) => throw new NotImplementedException("Not available in compute worker");
public Task<bool> DeleteBacktestsByIdsForUserAsync(User user, IEnumerable<string> ids) => throw new NotImplementedException("Not available in compute worker");
public bool DeleteBacktestsByUser(User user) => throw new NotImplementedException("Not available in compute worker");
public (IEnumerable<LightBacktest> Backtests, int TotalCount) GetBacktestsByUserPaginated(User user, int page, int pageSize, BacktestSortableColumn sortBy, string sortOrder = "desc", BacktestsFilter? filter = null) => throw new NotImplementedException("Not available in compute worker");
public Task<(IEnumerable<LightBacktest> Backtests, int TotalCount)> GetBacktestsByUserPaginatedAsync(User user, int page, int pageSize, BacktestSortableColumn sortBy, string sortOrder = "desc", BacktestsFilter? filter = null) => throw new NotImplementedException("Not available in compute worker");
public Task<bool> DeleteBacktestsByRequestIdAsync(Guid requestId) => throw new NotImplementedException("Not available in compute worker");
public Task<int> DeleteBacktestsByFiltersAsync(User user, BacktestsFilter filter) => throw new NotImplementedException("Not available in compute worker");
public Task InsertBundleBacktestRequestForUserAsync(User user, BundleBacktestRequest bundleRequest, bool saveAsTemplate = false) => throw new NotImplementedException("Not available in compute worker");
public IEnumerable<BundleBacktestRequest> GetBundleBacktestRequestsByUser(User user) => throw new NotImplementedException("Not available in compute worker");
public Task<IEnumerable<BundleBacktestRequest>> GetBundleBacktestRequestsByUserAsync(User user) => throw new NotImplementedException("Not available in compute worker");
public BundleBacktestRequest? GetBundleBacktestRequestByIdForUser(User user, Guid id) => throw new NotImplementedException("Not available in compute worker");
public Task<BundleBacktestRequest?> GetBundleBacktestRequestByIdForUserAsync(User user, Guid id) => throw new NotImplementedException("Not available in compute worker");
public void UpdateBundleBacktestRequest(BundleBacktestRequest bundleRequest) => throw new NotImplementedException("Not available in compute worker");
public Task UpdateBundleBacktestRequestAsync(BundleBacktestRequest bundleRequest) => throw new NotImplementedException("Not available in compute worker");
public void DeleteBundleBacktestRequestByIdForUser(User user, Guid id) => throw new NotImplementedException("Not available in compute worker");
public Task DeleteBundleBacktestRequestByIdForUserAsync(User user, Guid id) => throw new NotImplementedException("Not available in compute worker");
public IEnumerable<BundleBacktestRequest> GetBundleBacktestRequestsByStatus(BundleBacktestRequestStatus status) => throw new NotImplementedException("Not available in compute worker");
public Task<IEnumerable<BundleBacktestRequest>> GetBundleBacktestRequestsByStatusAsync(BundleBacktestRequestStatus status) => throw new NotImplementedException("Not available in compute worker");
}

View File

@@ -15,18 +15,18 @@ namespace Managing.Application.Backtests;
/// <summary> /// <summary>
/// Service for creating and managing backtest jobs in the queue /// Service for creating and managing backtest jobs in the queue
/// </summary> /// </summary>
public class BacktestJobService public class JobService
{ {
private readonly IBacktestJobRepository _jobRepository; private readonly IJobRepository _jobRepository;
private readonly IBacktestRepository _backtestRepository; private readonly IBacktestRepository _backtestRepository;
private readonly IKaigenService _kaigenService; private readonly IKaigenService _kaigenService;
private readonly ILogger<BacktestJobService> _logger; private readonly ILogger<JobService> _logger;
public BacktestJobService( public JobService(
IBacktestJobRepository jobRepository, IJobRepository jobRepository,
IBacktestRepository backtestRepository, IBacktestRepository backtestRepository,
IKaigenService kaigenService, IKaigenService kaigenService,
ILogger<BacktestJobService> logger) ILogger<JobService> logger)
{ {
_jobRepository = jobRepository; _jobRepository = jobRepository;
_backtestRepository = backtestRepository; _backtestRepository = backtestRepository;
@@ -37,7 +37,7 @@ public class BacktestJobService
/// <summary> /// <summary>
/// Creates a single backtest job /// Creates a single backtest job
/// </summary> /// </summary>
public async Task<BacktestJob> CreateJobAsync( public async Task<Job> CreateJobAsync(
TradingBotConfig config, TradingBotConfig config,
DateTime startDate, DateTime startDate,
DateTime endDate, DateTime endDate,
@@ -63,10 +63,10 @@ public class BacktestJobService
try try
{ {
var job = new BacktestJob var job = new Job
{ {
UserId = user.Id, UserId = user.Id,
Status = BacktestJobStatus.Pending, Status = JobStatus.Pending,
JobType = JobType.Backtest, JobType = JobType.Backtest,
Priority = priority, Priority = priority,
ConfigJson = JsonSerializer.Serialize(config), ConfigJson = JsonSerializer.Serialize(config),
@@ -109,11 +109,11 @@ public class BacktestJobService
/// <summary> /// <summary>
/// Creates multiple backtest jobs from bundle variants /// Creates multiple backtest jobs from bundle variants
/// </summary> /// </summary>
public async Task<List<BacktestJob>> CreateBundleJobsAsync( public async Task<List<Job>> CreateBundleJobsAsync(
BundleBacktestRequest bundleRequest, BundleBacktestRequest bundleRequest,
List<RunBacktestRequest> backtestRequests) List<RunBacktestRequest> backtestRequests)
{ {
var jobs = new List<BacktestJob>(); var jobs = new List<Job>();
var creditRequestId = (string?)null; var creditRequestId = (string?)null;
try try
@@ -203,10 +203,10 @@ public class BacktestJobService
UseForDynamicStopLoss = backtestRequest.Config.UseForDynamicStopLoss UseForDynamicStopLoss = backtestRequest.Config.UseForDynamicStopLoss
}; };
var job = new BacktestJob var job = new Job
{ {
UserId = bundleRequest.User.Id, UserId = bundleRequest.User.Id,
Status = BacktestJobStatus.Pending, Status = JobStatus.Pending,
JobType = JobType.Backtest, JobType = JobType.Backtest,
Priority = 0, // All bundle jobs have same priority Priority = 0, // All bundle jobs have same priority
ConfigJson = JsonSerializer.Serialize(backtestConfig), ConfigJson = JsonSerializer.Serialize(backtestConfig),

View File

@@ -29,7 +29,7 @@ namespace Managing.Application.Backtests
private readonly IMessengerService _messengerService; private readonly IMessengerService _messengerService;
private readonly IKaigenService _kaigenService; private readonly IKaigenService _kaigenService;
private readonly IHubContext<BacktestHub> _hubContext; private readonly IHubContext<BacktestHub> _hubContext;
private readonly BacktestJobService _jobService; private readonly JobService _jobService;
public Backtester( public Backtester(
IExchangeService exchangeService, IExchangeService exchangeService,
@@ -41,7 +41,7 @@ namespace Managing.Application.Backtests
IKaigenService kaigenService, IKaigenService kaigenService,
IHubContext<BacktestHub> hubContext, IHubContext<BacktestHub> hubContext,
IServiceScopeFactory serviceScopeFactory, IServiceScopeFactory serviceScopeFactory,
BacktestJobService jobService) JobService jobService)
{ {
_exchangeService = exchangeService; _exchangeService = exchangeService;
_backtestRepository = backtestRepository; _backtestRepository = backtestRepository;

View File

@@ -0,0 +1,149 @@
using Managing.Application.Abstractions.Services;
using Managing.Domain.Backtests;
using Microsoft.Extensions.Logging;
namespace Managing.Application.Backtests;
/// <summary>
/// Service for executing genetic algorithm requests without Orleans dependencies.
/// Extracted from GeneticBacktestGrain to be reusable in compute workers.
/// </summary>
public class GeneticExecutor
{
private readonly ILogger<GeneticExecutor> _logger;
private readonly IGeneticService _geneticService;
private readonly IAccountService _accountService;
private readonly IWebhookService _webhookService;
public GeneticExecutor(
ILogger<GeneticExecutor> logger,
IGeneticService geneticService,
IAccountService accountService,
IWebhookService webhookService)
{
_logger = logger;
_geneticService = geneticService;
_accountService = accountService;
_webhookService = webhookService;
}
/// <summary>
/// Executes a genetic algorithm request.
/// </summary>
/// <param name="geneticRequestId">The genetic request ID to process</param>
/// <param name="progressCallback">Optional callback for progress updates (0-100)</param>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns>The genetic algorithm result</returns>
public async Task<GeneticAlgorithmResult> ExecuteAsync(
string geneticRequestId,
Func<int, Task> progressCallback = null,
CancellationToken cancellationToken = default)
{
try
{
// Load the request by status lists and filter by ID (Pending first, then Failed for retries)
var pending = await _geneticService.GetGeneticRequestsAsync(GeneticRequestStatus.Pending);
var failed = await _geneticService.GetGeneticRequestsAsync(GeneticRequestStatus.Failed);
var request = pending.Concat(failed).FirstOrDefault(r => r.RequestId == geneticRequestId);
if (request == null)
{
_logger.LogWarning("[GeneticExecutor] Request {RequestId} not found among pending/failed.",
geneticRequestId);
throw new InvalidOperationException($"Genetic request {geneticRequestId} not found");
}
_logger.LogInformation("[GeneticExecutor] Processing genetic request {RequestId} for user {UserId}",
request.RequestId, request.User.Id);
// Mark running
request.Status = GeneticRequestStatus.Running;
await _geneticService.UpdateGeneticRequestAsync(request);
// Load user accounts if not already loaded
if (request.User.Accounts == null || !request.User.Accounts.Any())
{
request.User.Accounts = (await _accountService.GetAccountsByUserAsync(request.User)).ToList();
}
// Create progress wrapper if callback provided
Func<int, Task> wrappedProgressCallback = null;
if (progressCallback != null)
{
wrappedProgressCallback = async (percentage) =>
{
try
{
await progressCallback(percentage);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error in progress callback for genetic request {RequestId}", geneticRequestId);
}
};
}
// Run GA
var result = await _geneticService.RunGeneticAlgorithm(request, cancellationToken);
// Update final state
request.Status = GeneticRequestStatus.Completed;
request.CompletedAt = DateTime.UtcNow;
request.BestFitness = result.BestFitness;
request.BestIndividual = result.BestIndividual;
request.ProgressInfo = result.ProgressInfo;
await _geneticService.UpdateGeneticRequestAsync(request);
_logger.LogInformation("[GeneticExecutor] Completed genetic request {RequestId}. Best Fitness: {BestFitness}",
request.RequestId, result.BestFitness);
// Send webhook notification if user has telegram channel
if (!string.IsNullOrEmpty(request.User?.TelegramChannel))
{
var message = $"✅ Genetic algorithm optimization completed for {request.Ticker} on {request.Timeframe}. " +
$"Request ID: {request.RequestId}. " +
$"Best Fitness: {result.BestFitness:F4}. " +
$"Generations: {request.Generations}, Population: {request.PopulationSize}.";
await _webhookService.SendMessage(message, request.User.TelegramChannel);
}
return result;
}
catch (Exception ex)
{
_logger.LogError(ex, "[GeneticExecutor] Error processing genetic request {RequestId}", geneticRequestId);
// Try to mark as failed
try
{
var running = await _geneticService.GetGeneticRequestsAsync(GeneticRequestStatus.Running);
var req = running.FirstOrDefault(r => r.RequestId == geneticRequestId);
if (req != null)
{
req.Status = GeneticRequestStatus.Failed;
req.ErrorMessage = ex.Message;
req.CompletedAt = DateTime.UtcNow;
await _geneticService.UpdateGeneticRequestAsync(req);
// Send webhook notification for failed genetic request
if (!string.IsNullOrEmpty(req.User?.TelegramChannel))
{
var message = $"❌ Genetic algorithm optimization failed for {req.Ticker} on {req.Timeframe}. " +
$"Request ID: {req.RequestId}. " +
$"Error: {ex.Message}";
await _webhookService.SendMessage(message, req.User.TelegramChannel);
}
}
}
catch (Exception updateEx)
{
_logger.LogError(updateEx, "[GeneticExecutor] Failed to update request status to Failed for {RequestId}", geneticRequestId);
}
throw;
}
}
}

View File

@@ -347,7 +347,7 @@ public class GeneticService : IGeneticService
CrossoverProbability = 0.75f, // Fixed crossover rate as in frontend CrossoverProbability = 0.75f, // Fixed crossover rate as in frontend
TaskExecutor = new ParallelTaskExecutor TaskExecutor = new ParallelTaskExecutor
{ {
MinThreads = 4, MinThreads = 2,
MaxThreads = Environment.ProcessorCount MaxThreads = Environment.ProcessorCount
} }
}; };

View File

@@ -77,10 +77,10 @@ public class BacktestComputeWorker : BackgroundService
try try
{ {
using var scope = _scopeFactory.CreateScope(); using var scope = _scopeFactory.CreateScope();
var jobRepository = scope.ServiceProvider.GetRequiredService<IBacktestJobRepository>(); var jobRepository = scope.ServiceProvider.GetRequiredService<IJobRepository>();
// Try to claim a job // Try to claim a backtest job (exclude genetic jobs)
var job = await jobRepository.ClaimNextJobAsync(_options.WorkerId); var job = await jobRepository.ClaimNextJobAsync(_options.WorkerId, JobType.Backtest);
if (job == null) if (job == null)
{ {
@@ -114,11 +114,11 @@ public class BacktestComputeWorker : BackgroundService
} }
private async Task ProcessJobAsync( private async Task ProcessJobAsync(
BacktestJob job, Job job,
CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
using var scope = _scopeFactory.CreateScope(); using var scope = _scopeFactory.CreateScope();
var jobRepository = scope.ServiceProvider.GetRequiredService<IBacktestJobRepository>(); var jobRepository = scope.ServiceProvider.GetRequiredService<IJobRepository>();
var executor = scope.ServiceProvider.GetRequiredService<BacktestExecutor>(); var executor = scope.ServiceProvider.GetRequiredService<BacktestExecutor>();
var userService = scope.ServiceProvider.GetRequiredService<IUserService>(); var userService = scope.ServiceProvider.GetRequiredService<IUserService>();
var exchangeService = scope.ServiceProvider.GetRequiredService<IExchangeService>(); var exchangeService = scope.ServiceProvider.GetRequiredService<IExchangeService>();
@@ -184,7 +184,7 @@ public class BacktestComputeWorker : BackgroundService
progressCallback: progressCallback); progressCallback: progressCallback);
// Update job with result // Update job with result
job.Status = BacktestJobStatus.Completed; job.Status = JobStatus.Completed;
job.ProgressPercentage = 100; job.ProgressPercentage = 100;
job.ResultJson = JsonSerializer.Serialize(result); job.ResultJson = JsonSerializer.Serialize(result);
job.CompletedAt = DateTime.UtcNow; job.CompletedAt = DateTime.UtcNow;
@@ -207,24 +207,7 @@ public class BacktestComputeWorker : BackgroundService
_logger.LogError(ex, "Error processing backtest job {JobId}", job.Id); _logger.LogError(ex, "Error processing backtest job {JobId}", job.Id);
SentrySdk.CaptureException(ex); SentrySdk.CaptureException(ex);
// Update job status to failed await HandleJobFailure(job, ex, jobRepository, scope.ServiceProvider);
try
{
job.Status = BacktestJobStatus.Failed;
job.ErrorMessage = ex.Message;
job.CompletedAt = DateTime.UtcNow;
await jobRepository.UpdateAsync(job);
// Update bundle request if this is part of a bundle
if (job.BundleRequestId.HasValue)
{
await UpdateBundleRequestProgress(job.BundleRequestId.Value, scope.ServiceProvider);
}
}
catch (Exception updateEx)
{
_logger.LogError(updateEx, "Error updating job {JobId} status to failed", job.Id);
}
} }
} }
@@ -233,14 +216,15 @@ public class BacktestComputeWorker : BackgroundService
try try
{ {
var backtestRepository = serviceProvider.GetRequiredService<IBacktestRepository>(); var backtestRepository = serviceProvider.GetRequiredService<IBacktestRepository>();
var jobRepository = serviceProvider.GetRequiredService<IBacktestJobRepository>(); var jobRepository = serviceProvider.GetRequiredService<IJobRepository>();
var userService = serviceProvider.GetRequiredService<IUserService>(); var userService = serviceProvider.GetRequiredService<IUserService>();
var webhookService = serviceProvider.GetRequiredService<IWebhookService>();
// Get all jobs for this bundle // Get all jobs for this bundle
var jobs = await jobRepository.GetByBundleRequestIdAsync(bundleRequestId); var jobs = await jobRepository.GetByBundleRequestIdAsync(bundleRequestId);
var completedJobs = jobs.Count(j => j.Status == BacktestJobStatus.Completed); var completedJobs = jobs.Count(j => j.Status == JobStatus.Completed);
var failedJobs = jobs.Count(j => j.Status == BacktestJobStatus.Failed); var failedJobs = jobs.Count(j => j.Status == JobStatus.Failed);
var runningJobs = jobs.Count(j => j.Status == BacktestJobStatus.Running); var runningJobs = jobs.Count(j => j.Status == JobStatus.Running);
var totalJobs = jobs.Count(); var totalJobs = jobs.Count();
if (totalJobs == 0) if (totalJobs == 0)
@@ -265,6 +249,8 @@ public class BacktestComputeWorker : BackgroundService
return; return;
} }
var previousStatus = bundleRequest.Status;
// Update bundle request progress // Update bundle request progress
bundleRequest.CompletedBacktests = completedJobs; bundleRequest.CompletedBacktests = completedJobs;
bundleRequest.FailedBacktests = failedJobs; bundleRequest.FailedBacktests = failedJobs;
@@ -298,7 +284,7 @@ public class BacktestComputeWorker : BackgroundService
// Update results list from completed jobs // Update results list from completed jobs
var completedJobResults = jobs var completedJobResults = jobs
.Where(j => j.Status == BacktestJobStatus.Completed && !string.IsNullOrEmpty(j.ResultJson)) .Where(j => j.Status == JobStatus.Completed && !string.IsNullOrEmpty(j.ResultJson))
.Select(j => .Select(j =>
{ {
try try
@@ -318,6 +304,28 @@ public class BacktestComputeWorker : BackgroundService
await backtestRepository.UpdateBundleBacktestRequestAsync(bundleRequest); await backtestRepository.UpdateBundleBacktestRequestAsync(bundleRequest);
// Send webhook notification if bundle request just completed
if (previousStatus != BundleBacktestRequestStatus.Completed &&
bundleRequest.Status == BundleBacktestRequestStatus.Completed &&
!string.IsNullOrEmpty(user.TelegramChannel))
{
var message = $"✅ Bundle backtest '{bundleRequest.Name}' (ID: {bundleRequest.RequestId}) completed successfully. " +
$"Completed: {completedJobs}/{totalJobs} backtests" +
(failedJobs > 0 ? $", Failed: {failedJobs}" : "") +
$". Results: {completedJobResults.Count} backtest(s) generated.";
await webhookService.SendMessage(message, user.TelegramChannel);
}
else if (previousStatus != BundleBacktestRequestStatus.Failed &&
bundleRequest.Status == BundleBacktestRequestStatus.Failed &&
!string.IsNullOrEmpty(user.TelegramChannel))
{
var message = $"❌ Bundle backtest '{bundleRequest.Name}' (ID: {bundleRequest.RequestId}) failed. " +
$"All {totalJobs} backtests failed. Error: {bundleRequest.ErrorMessage}";
await webhookService.SendMessage(message, user.TelegramChannel);
}
_logger.LogInformation( _logger.LogInformation(
"Updated bundle request {BundleRequestId} progress: {Completed}/{Total} completed, {Failed} failed, {Running} running", "Updated bundle request {BundleRequestId} progress: {Completed}/{Total} completed, {Failed} failed, {Running} running",
bundleRequestId, completedJobs, totalJobs, failedJobs, runningJobs); bundleRequestId, completedJobs, totalJobs, failedJobs, runningJobs);
@@ -337,13 +345,58 @@ public class BacktestComputeWorker : BackgroundService
await Task.Delay(TimeSpan.FromMinutes(1), cancellationToken); // Check every minute await Task.Delay(TimeSpan.FromMinutes(1), cancellationToken); // Check every minute
using var scope = _scopeFactory.CreateScope(); using var scope = _scopeFactory.CreateScope();
var jobRepository = scope.ServiceProvider.GetRequiredService<IBacktestJobRepository>(); var jobRepository = scope.ServiceProvider.GetRequiredService<IJobRepository>();
var resetCount = await jobRepository.ResetStaleJobsAsync(_options.StaleJobTimeoutMinutes); // Get stale jobs for this worker
var runningJobs = await jobRepository.GetRunningJobsByWorkerIdAsync(_options.WorkerId);
var staleJobs = runningJobs
.Where(j => j.JobType == JobType.Backtest &&
(j.LastHeartbeat == null ||
j.LastHeartbeat < DateTime.UtcNow.AddMinutes(-_options.StaleJobTimeoutMinutes)))
.ToList();
if (resetCount > 0) foreach (var job in staleJobs)
{ {
_logger.LogInformation("Reset {Count} stale backtest jobs back to Pending status", resetCount); // If it's stale but retryable, reset to pending with retry count
if (job.RetryCount < job.MaxRetries)
{
job.Status = JobStatus.Pending;
job.RetryCount++;
var backoffMinutes = Math.Min(Math.Pow(2, job.RetryCount), _options.MaxRetryDelayMinutes);
job.RetryAfter = DateTime.UtcNow.AddMinutes(backoffMinutes);
job.ErrorMessage = $"Worker timeout - retry {job.RetryCount}/{job.MaxRetries}";
job.FailureCategory = FailureCategory.SystemError;
_logger.LogWarning(
"Stale job {JobId} will be retried (attempt {RetryCount}/{MaxRetries}) after {RetryAfter}",
job.Id, job.RetryCount, job.MaxRetries, job.RetryAfter);
}
else
{
// Exceeded retries - mark as failed
job.Status = JobStatus.Failed;
job.ErrorMessage = "Worker timeout - exceeded max retries";
job.FailureCategory = FailureCategory.SystemError;
job.IsRetryable = false;
job.CompletedAt = DateTime.UtcNow;
// Notify permanent failure
await NotifyPermanentFailure(job, new TimeoutException("Worker timeout"), scope.ServiceProvider);
// Update bundle request if this is part of a bundle
if (job.BundleRequestId.HasValue)
{
await UpdateBundleRequestProgress(job.BundleRequestId.Value, scope.ServiceProvider);
}
}
job.AssignedWorkerId = null;
job.LastHeartbeat = null;
await jobRepository.UpdateAsync(job);
}
if (staleJobs.Count > 0)
{
_logger.LogInformation("Processed {Count} stale backtest jobs", staleJobs.Count);
} }
} }
catch (Exception ex) catch (Exception ex)
@@ -362,7 +415,7 @@ public class BacktestComputeWorker : BackgroundService
await Task.Delay(TimeSpan.FromSeconds(_options.HeartbeatIntervalSeconds), cancellationToken); await Task.Delay(TimeSpan.FromSeconds(_options.HeartbeatIntervalSeconds), cancellationToken);
using var scope = _scopeFactory.CreateScope(); using var scope = _scopeFactory.CreateScope();
var jobRepository = scope.ServiceProvider.GetRequiredService<IBacktestJobRepository>(); var jobRepository = scope.ServiceProvider.GetRequiredService<IJobRepository>();
// Update heartbeat for all jobs assigned to this worker // Update heartbeat for all jobs assigned to this worker
var runningJobs = await jobRepository.GetRunningJobsByWorkerIdAsync(_options.WorkerId); var runningJobs = await jobRepository.GetRunningJobsByWorkerIdAsync(_options.WorkerId);
@@ -380,6 +433,118 @@ public class BacktestComputeWorker : BackgroundService
} }
} }
private async Task HandleJobFailure(
Job job,
Exception ex,
IJobRepository jobRepository,
IServiceProvider serviceProvider)
{
try
{
// Categorize the failure
var failureCategory = CategorizeFailure(ex);
var isRetryable = IsRetryableFailure(ex, failureCategory);
// Check if we should retry
if (isRetryable && job.RetryCount < job.MaxRetries)
{
// Calculate exponential backoff: 2^retryCount minutes, capped at MaxRetryDelayMinutes
var backoffMinutes = Math.Min(Math.Pow(2, job.RetryCount), _options.MaxRetryDelayMinutes);
job.RetryAfter = DateTime.UtcNow.AddMinutes(backoffMinutes);
job.RetryCount++;
job.Status = JobStatus.Pending; // Reset to pending for retry
job.AssignedWorkerId = null; // Allow any worker to pick it up
job.ErrorMessage = $"Retry {job.RetryCount}/{job.MaxRetries}: {ex.Message}";
job.FailureCategory = failureCategory;
job.IsRetryable = true;
_logger.LogWarning(
"Job {JobId} will be retried (attempt {RetryCount}/{MaxRetries}) after {RetryAfter}. Error: {Error}",
job.Id, job.RetryCount, job.MaxRetries, job.RetryAfter, ex.Message);
}
else
{
// Permanent failure - mark as failed
job.Status = JobStatus.Failed;
job.ErrorMessage = ex.Message;
job.FailureCategory = failureCategory;
job.IsRetryable = false;
job.CompletedAt = DateTime.UtcNow;
_logger.LogError(
"Job {JobId} failed permanently after {RetryCount} retries. Error: {Error}",
job.Id, job.RetryCount, ex.Message);
// Send notification for permanent failure
await NotifyPermanentFailure(job, ex, serviceProvider);
// Update bundle request if this is part of a bundle
if (job.BundleRequestId.HasValue)
{
await UpdateBundleRequestProgress(job.BundleRequestId.Value, serviceProvider);
}
}
job.LastHeartbeat = DateTime.UtcNow;
await jobRepository.UpdateAsync(job);
}
catch (Exception updateEx)
{
_logger.LogError(updateEx, "Failed to update job {JobId} status after failure", job.Id);
}
}
private FailureCategory CategorizeFailure(Exception ex)
{
return ex switch
{
TimeoutException => FailureCategory.Transient,
TaskCanceledException => FailureCategory.Transient,
HttpRequestException => FailureCategory.Transient,
InvalidOperationException when ex.Message.Contains("candles") || ex.Message.Contains("No candles") => FailureCategory.DataError,
InvalidOperationException when ex.Message.Contains("User") || ex.Message.Contains("not found") => FailureCategory.UserError,
OutOfMemoryException => FailureCategory.SystemError,
_ => FailureCategory.Unknown
};
}
private bool IsRetryableFailure(Exception ex, FailureCategory category)
{
// Don't retry user errors or data errors (missing candles, invalid config)
if (category == FailureCategory.UserError || category == FailureCategory.DataError)
return false;
// Retry transient and system errors
return category == FailureCategory.Transient || category == FailureCategory.SystemError;
}
private async Task NotifyPermanentFailure(
Job job,
Exception ex,
IServiceProvider serviceProvider)
{
try
{
var webhookService = serviceProvider.GetRequiredService<IWebhookService>();
const string alertsChannel = "2676086723";
var jobTypeName = job.JobType == JobType.Genetic ? "Genetic" : "Backtest";
var message = $"🚨 **{jobTypeName} Job Failed Permanently**\n" +
$"Job ID: `{job.Id}`\n" +
$"User ID: {job.UserId}\n" +
$"Retry Attempts: {job.RetryCount}/{job.MaxRetries}\n" +
$"Failure Category: {job.FailureCategory}\n" +
$"Error: {ex.Message}\n" +
$"Time: {DateTime.UtcNow:yyyy-MM-dd HH:mm:ss} UTC";
await webhookService.SendMessage(message, alertsChannel);
}
catch (Exception notifyEx)
{
_logger.LogError(notifyEx, "Failed to send permanent failure notification for job {JobId}", job.Id);
}
}
public override void Dispose() public override void Dispose()
{ {
_semaphore?.Dispose(); _semaphore?.Dispose();
@@ -418,5 +583,15 @@ public class BacktestComputeWorkerOptions
/// Timeout in minutes for considering a job stale /// Timeout in minutes for considering a job stale
/// </summary> /// </summary>
public int StaleJobTimeoutMinutes { get; set; } = 5; public int StaleJobTimeoutMinutes { get; set; } = 5;
/// <summary>
/// Default maximum retry attempts for failed jobs
/// </summary>
public int DefaultMaxRetries { get; set; } = 3;
/// <summary>
/// Maximum retry delay in minutes (cap for exponential backoff)
/// </summary>
public int MaxRetryDelayMinutes { get; set; } = 60;
} }

View File

@@ -0,0 +1,429 @@
using System.Text.Json;
using Managing.Application.Abstractions.Repositories;
using Managing.Application.Abstractions.Services;
using Managing.Application.Backtests;
using Managing.Domain.Backtests;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using static Managing.Common.Enums;
namespace Managing.Application.Workers;
/// <summary>
/// Background worker that processes genetic algorithm jobs from the queue.
/// Polls for pending genetic jobs, claims them using advisory locks, and processes them.
/// </summary>
public class GeneticComputeWorker : BackgroundService
{
private readonly IServiceScopeFactory _scopeFactory;
private readonly ILogger<GeneticComputeWorker> _logger;
private readonly GeneticComputeWorkerOptions _options;
private readonly SemaphoreSlim _semaphore;
public GeneticComputeWorker(
IServiceScopeFactory scopeFactory,
ILogger<GeneticComputeWorker> logger,
IOptions<GeneticComputeWorkerOptions> options)
{
_scopeFactory = scopeFactory;
_logger = logger;
_options = options.Value;
_semaphore = new SemaphoreSlim(_options.MaxConcurrentGenetics, _options.MaxConcurrentGenetics);
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation(
"GeneticComputeWorker starting. WorkerId: {WorkerId}, MaxConcurrent: {MaxConcurrent}, PollInterval: {PollInterval}s",
_options.WorkerId, _options.MaxConcurrentGenetics, _options.JobPollIntervalSeconds);
// Background task for stale job recovery
var staleJobRecoveryTask = Task.Run(() => StaleJobRecoveryLoop(stoppingToken), stoppingToken);
// Background task for heartbeat updates
var heartbeatTask = Task.Run(() => HeartbeatLoop(stoppingToken), stoppingToken);
// Main job processing loop
while (!stoppingToken.IsCancellationRequested)
{
try
{
await ProcessJobsAsync(stoppingToken);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error in GeneticComputeWorker main loop");
SentrySdk.CaptureException(ex);
}
await Task.Delay(TimeSpan.FromSeconds(_options.JobPollIntervalSeconds), stoppingToken);
}
_logger.LogInformation("GeneticComputeWorker stopping");
}
private async Task ProcessJobsAsync(CancellationToken cancellationToken)
{
// Check if we have capacity
if (!await _semaphore.WaitAsync(0, cancellationToken))
{
// At capacity, skip this iteration
return;
}
try
{
using var scope = _scopeFactory.CreateScope();
var jobRepository = scope.ServiceProvider.GetRequiredService<IJobRepository>();
// Try to claim a genetic job
var job = await jobRepository.ClaimNextJobAsync(_options.WorkerId, JobType.Genetic);
if (job == null)
{
// No jobs available, release semaphore
_semaphore.Release();
return;
}
_logger.LogInformation("Claimed genetic job {JobId} for worker {WorkerId}", job.Id, _options.WorkerId);
// Process the job asynchronously (don't await, let it run in background)
// Create a new scope for the job processing to ensure proper lifetime management
_ = Task.Run(async () =>
{
try
{
await ProcessJobAsync(job, cancellationToken);
}
finally
{
_semaphore.Release();
}
}, cancellationToken);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error claiming or processing genetic job");
_semaphore.Release();
throw;
}
}
private async Task ProcessJobAsync(
Job job,
CancellationToken cancellationToken)
{
using var scope = _scopeFactory.CreateScope();
var jobRepository = scope.ServiceProvider.GetRequiredService<IJobRepository>();
var geneticExecutor = scope.ServiceProvider.GetRequiredService<GeneticExecutor>();
try
{
_logger.LogInformation(
"Processing genetic job {JobId} (GeneticRequestId: {GeneticRequestId}, UserId: {UserId})",
job.Id, job.GeneticRequestId, job.UserId);
if (string.IsNullOrEmpty(job.GeneticRequestId))
{
throw new InvalidOperationException("GeneticRequestId is required for genetic jobs");
}
// Progress callback to update job progress
Func<int, Task> progressCallback = async (percentage) =>
{
try
{
job.ProgressPercentage = percentage;
job.LastHeartbeat = DateTime.UtcNow;
await jobRepository.UpdateAsync(job);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error updating job progress for job {JobId}", job.Id);
}
};
// Execute the genetic algorithm
var result = await geneticExecutor.ExecuteAsync(
job.GeneticRequestId,
progressCallback,
cancellationToken);
// Update job with result
job.Status = JobStatus.Completed;
job.ProgressPercentage = 100;
job.ResultJson = JsonSerializer.Serialize(new
{
BestFitness = result.BestFitness,
BestIndividual = result.BestIndividual,
ProgressInfo = result.ProgressInfo
});
job.CompletedAt = DateTime.UtcNow;
job.LastHeartbeat = DateTime.UtcNow;
await jobRepository.UpdateAsync(job);
_logger.LogInformation(
"Completed genetic job {JobId}. Best Fitness: {BestFitness}",
job.Id, result.BestFitness);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error processing genetic job {JobId}", job.Id);
SentrySdk.CaptureException(ex);
await HandleJobFailure(job, ex, jobRepository, scope.ServiceProvider);
}
}
private async Task StaleJobRecoveryLoop(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
try
{
await Task.Delay(TimeSpan.FromMinutes(1), cancellationToken); // Check every minute
using var scope = _scopeFactory.CreateScope();
var jobRepository = scope.ServiceProvider.GetRequiredService<IJobRepository>();
// Reset stale genetic jobs only
var runningJobs = await jobRepository.GetRunningJobsByWorkerIdAsync(_options.WorkerId);
var staleJobs = runningJobs
.Where(j => j.JobType == JobType.Genetic &&
(j.LastHeartbeat == null ||
j.LastHeartbeat < DateTime.UtcNow.AddMinutes(-_options.StaleJobTimeoutMinutes)))
.ToList();
foreach (var job in staleJobs)
{
// If it's stale but retryable, reset to pending with retry count
if (job.RetryCount < job.MaxRetries)
{
job.Status = JobStatus.Pending;
job.RetryCount++;
var backoffMinutes = Math.Min(Math.Pow(2, job.RetryCount), _options.MaxRetryDelayMinutes);
job.RetryAfter = DateTime.UtcNow.AddMinutes(backoffMinutes);
job.ErrorMessage = $"Worker timeout - retry {job.RetryCount}/{job.MaxRetries}";
job.FailureCategory = FailureCategory.SystemError;
_logger.LogWarning(
"Stale job {JobId} will be retried (attempt {RetryCount}/{MaxRetries}) after {RetryAfter}",
job.Id, job.RetryCount, job.MaxRetries, job.RetryAfter);
}
else
{
// Exceeded retries - mark as failed
job.Status = JobStatus.Failed;
job.ErrorMessage = "Worker timeout - exceeded max retries";
job.FailureCategory = FailureCategory.SystemError;
job.IsRetryable = false;
job.CompletedAt = DateTime.UtcNow;
// Notify permanent failure
await NotifyPermanentFailure(job, new TimeoutException("Worker timeout"), scope.ServiceProvider);
}
job.AssignedWorkerId = null;
job.LastHeartbeat = null;
await jobRepository.UpdateAsync(job);
}
if (staleJobs.Count > 0)
{
_logger.LogInformation("Processed {Count} stale genetic jobs", staleJobs.Count);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error in stale job recovery loop");
}
}
}
private async Task HeartbeatLoop(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
try
{
await Task.Delay(TimeSpan.FromSeconds(_options.HeartbeatIntervalSeconds), cancellationToken);
using var scope = _scopeFactory.CreateScope();
var jobRepository = scope.ServiceProvider.GetRequiredService<IJobRepository>();
// Update heartbeat for all genetic jobs assigned to this worker
var runningJobs = await jobRepository.GetRunningJobsByWorkerIdAsync(_options.WorkerId);
var geneticJobs = runningJobs.Where(j => j.JobType == JobType.Genetic);
foreach (var job in geneticJobs)
{
job.LastHeartbeat = DateTime.UtcNow;
await jobRepository.UpdateAsync(job);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error in heartbeat loop");
}
}
}
private async Task HandleJobFailure(
Job job,
Exception ex,
IJobRepository jobRepository,
IServiceProvider serviceProvider)
{
try
{
// Categorize the failure
var failureCategory = CategorizeFailure(ex);
var isRetryable = IsRetryableFailure(ex, failureCategory);
// Check if we should retry
if (isRetryable && job.RetryCount < job.MaxRetries)
{
// Calculate exponential backoff: 2^retryCount minutes, capped at MaxRetryDelayMinutes
var backoffMinutes = Math.Min(Math.Pow(2, job.RetryCount), _options.MaxRetryDelayMinutes);
job.RetryAfter = DateTime.UtcNow.AddMinutes(backoffMinutes);
job.RetryCount++;
job.Status = JobStatus.Pending; // Reset to pending for retry
job.AssignedWorkerId = null; // Allow any worker to pick it up
job.ErrorMessage = $"Retry {job.RetryCount}/{job.MaxRetries}: {ex.Message}";
job.FailureCategory = failureCategory;
job.IsRetryable = true;
_logger.LogWarning(
"Job {JobId} will be retried (attempt {RetryCount}/{MaxRetries}) after {RetryAfter}. Error: {Error}",
job.Id, job.RetryCount, job.MaxRetries, job.RetryAfter, ex.Message);
}
else
{
// Permanent failure - mark as failed
job.Status = JobStatus.Failed;
job.ErrorMessage = ex.Message;
job.FailureCategory = failureCategory;
job.IsRetryable = false;
job.CompletedAt = DateTime.UtcNow;
_logger.LogError(
"Job {JobId} failed permanently after {RetryCount} retries. Error: {Error}",
job.Id, job.RetryCount, ex.Message);
// Send notification for permanent failure
await NotifyPermanentFailure(job, ex, serviceProvider);
}
job.LastHeartbeat = DateTime.UtcNow;
await jobRepository.UpdateAsync(job);
}
catch (Exception updateEx)
{
_logger.LogError(updateEx, "Failed to update job {JobId} status after failure", job.Id);
}
}
private FailureCategory CategorizeFailure(Exception ex)
{
return ex switch
{
TimeoutException => FailureCategory.Transient,
TaskCanceledException => FailureCategory.Transient,
HttpRequestException => FailureCategory.Transient,
InvalidOperationException when ex.Message.Contains("candles") || ex.Message.Contains("data") => FailureCategory.DataError,
InvalidOperationException when ex.Message.Contains("User") || ex.Message.Contains("not found") => FailureCategory.UserError,
OutOfMemoryException => FailureCategory.SystemError,
_ => FailureCategory.Unknown
};
}
private bool IsRetryableFailure(Exception ex, FailureCategory category)
{
// Don't retry user errors or data errors (missing candles, invalid config)
if (category == FailureCategory.UserError || category == FailureCategory.DataError)
return false;
// Retry transient and system errors
return category == FailureCategory.Transient || category == FailureCategory.SystemError;
}
private async Task NotifyPermanentFailure(
Job job,
Exception ex,
IServiceProvider serviceProvider)
{
try
{
var webhookService = serviceProvider.GetRequiredService<IWebhookService>();
const string alertsChannel = "2676086723";
var jobTypeName = job.JobType == JobType.Genetic ? "Genetic" : "Backtest";
var message = $"🚨 **{jobTypeName} Job Failed Permanently**\n" +
$"Job ID: `{job.Id}`\n" +
$"User ID: {job.UserId}\n" +
$"Retry Attempts: {job.RetryCount}/{job.MaxRetries}\n" +
$"Failure Category: {job.FailureCategory}\n" +
$"Error: {ex.Message}\n" +
$"Time: {DateTime.UtcNow:yyyy-MM-dd HH:mm:ss} UTC";
await webhookService.SendMessage(message, alertsChannel);
}
catch (Exception notifyEx)
{
_logger.LogError(notifyEx, "Failed to send permanent failure notification for job {JobId}", job.Id);
}
}
public override void Dispose()
{
_semaphore?.Dispose();
base.Dispose();
}
}
/// <summary>
/// Configuration options for GeneticComputeWorker
/// </summary>
public class GeneticComputeWorkerOptions
{
public const string SectionName = "GeneticComputeWorker";
/// <summary>
/// Unique identifier for this worker instance
/// </summary>
public string WorkerId { get; set; } = Environment.MachineName + "-genetic";
/// <summary>
/// Maximum number of concurrent genetic algorithm jobs to process
/// </summary>
public int MaxConcurrentGenetics { get; set; } = 2;
/// <summary>
/// Interval in seconds between job polling attempts
/// </summary>
public int JobPollIntervalSeconds { get; set; } = 5;
/// <summary>
/// Interval in seconds between heartbeat updates
/// </summary>
public int HeartbeatIntervalSeconds { get; set; } = 30;
/// <summary>
/// Timeout in minutes for considering a job stale
/// </summary>
public int StaleJobTimeoutMinutes { get; set; } = 10;
/// <summary>
/// Default maximum retry attempts for failed jobs
/// </summary>
public int DefaultMaxRetries { get; set; } = 3;
/// <summary>
/// Maximum retry delay in minutes (cap for exponential backoff)
/// </summary>
public int MaxRetryDelayMinutes { get; set; } = 60;
}

View File

@@ -400,7 +400,7 @@ public static class ApiBootstrap
// Processors // Processors
services.AddTransient<IBacktester, Backtester>(); services.AddTransient<IBacktester, Backtester>();
services.AddTransient<BacktestJobService>(); services.AddTransient<JobService>();
services.AddTransient<IExchangeProcessor, EvmProcessor>(); services.AddTransient<IExchangeProcessor, EvmProcessor>();
services.AddTransient<ITradaoService, TradaoService>(); services.AddTransient<ITradaoService, TradaoService>();
@@ -443,7 +443,7 @@ public static class ApiBootstrap
services.AddTransient<IAccountRepository, PostgreSqlAccountRepository>(); services.AddTransient<IAccountRepository, PostgreSqlAccountRepository>();
services.AddTransient<IBacktestRepository, PostgreSqlBacktestRepository>(); services.AddTransient<IBacktestRepository, PostgreSqlBacktestRepository>();
services.AddTransient<IBacktestJobRepository, PostgreSqlJobRepository>(); services.AddTransient<IJobRepository, PostgreSqlJobRepository>();
services.AddTransient<IGeneticRepository, PostgreSqlGeneticRepository>(); services.AddTransient<IGeneticRepository, PostgreSqlGeneticRepository>();
services.AddTransient<ITradingRepository, PostgreSqlTradingRepository>(); services.AddTransient<ITradingRepository, PostgreSqlTradingRepository>();
services.AddTransient<ISettingsRepository, PostgreSqlSettingsRepository>(); services.AddTransient<ISettingsRepository, PostgreSqlSettingsRepository>();

View File

@@ -1,3 +1,4 @@
using Managing.Application;
using Managing.Application.Abstractions; using Managing.Application.Abstractions;
using Managing.Application.Abstractions.Repositories; using Managing.Application.Abstractions.Repositories;
using Managing.Application.Abstractions.Services; using Managing.Application.Abstractions.Services;
@@ -57,7 +58,6 @@ public static class ComputeBootstrap
// Services not needed for compute worker (depend on IBacktester/Orleans) // Services not needed for compute worker (depend on IBacktester/Orleans)
// services.AddScoped<IStatisticService, StatisticService>(); // Requires IBacktester // services.AddScoped<IStatisticService, StatisticService>(); // Requires IBacktester
// services.AddScoped<ISettingsService, SettingsService>(); // Requires IBacktester // services.AddScoped<ISettingsService, SettingsService>(); // Requires IBacktester
// services.AddScoped<IGeneticService, GeneticService>(); // Requires IBacktester
// services.AddScoped<IAgentService, AgentService>(); // May require Orleans // services.AddScoped<IAgentService, AgentService>(); // May require Orleans
// services.AddScoped<IBotService, BotService>(); // May require Orleans // services.AddScoped<IBotService, BotService>(); // May require Orleans
// services.AddScoped<IWorkerService, WorkerService>(); // May require Orleans // services.AddScoped<IWorkerService, WorkerService>(); // May require Orleans
@@ -66,6 +66,13 @@ public static class ComputeBootstrap
// Processors // Processors
// Note: IBacktester not needed for compute worker - BacktestExecutor is used directly // Note: IBacktester not needed for compute worker - BacktestExecutor is used directly
services.AddTransient<BacktestExecutor>(); services.AddTransient<BacktestExecutor>();
services.AddTransient<GeneticExecutor>();
// Adapter to make BacktestExecutor implement IBacktester (needed for GeneticService)
services.AddTransient<IBacktester, BacktestExecutorAdapter>();
// Genetic service (needed for GeneticExecutor)
services.AddScoped<IGeneticService, GeneticService>();
services.AddTransient<IExchangeProcessor, EvmProcessor>(); services.AddTransient<IExchangeProcessor, EvmProcessor>();
services.AddTransient<ITradaoService, TradaoService>(); services.AddTransient<ITradaoService, TradaoService>();
@@ -120,7 +127,7 @@ public static class ComputeBootstrap
// PostgreSql Repositories // PostgreSql Repositories
services.AddTransient<IAccountRepository, PostgreSqlAccountRepository>(); services.AddTransient<IAccountRepository, PostgreSqlAccountRepository>();
services.AddTransient<IBacktestRepository, PostgreSqlBacktestRepository>(); services.AddTransient<IBacktestRepository, PostgreSqlBacktestRepository>();
services.AddTransient<IBacktestJobRepository, PostgreSqlJobRepository>(); services.AddTransient<IJobRepository, PostgreSqlJobRepository>();
services.AddTransient<IGeneticRepository, PostgreSqlGeneticRepository>(); services.AddTransient<IGeneticRepository, PostgreSqlGeneticRepository>();
services.AddTransient<ITradingRepository, PostgreSqlTradingRepository>(); services.AddTransient<ITradingRepository, PostgreSqlTradingRepository>();
services.AddTransient<ISettingsRepository, PostgreSqlSettingsRepository>(); services.AddTransient<ISettingsRepository, PostgreSqlSettingsRepository>();

View File

@@ -564,6 +564,11 @@ public static class Enums
/// <summary> /// <summary>
/// Genetic algorithm backtest job /// Genetic algorithm backtest job
/// </summary> /// </summary>
GeneticBacktest GeneticBacktest,
/// <summary>
/// Genetic algorithm request processing job
/// </summary>
Genetic
} }
} }

View File

@@ -5,16 +5,16 @@ using static Managing.Common.Enums;
namespace Managing.Domain.Backtests; namespace Managing.Domain.Backtests;
/// <summary> /// <summary>
/// Represents a single backtest job in the queue system. /// Represents a single job in the queue system.
/// Can be a standalone backtest or part of a bundle backtest request. /// Can be a standalone backtest, genetic algorithm, or part of a bundle backtest request.
/// </summary> /// </summary>
public class BacktestJob public class Job
{ {
public BacktestJob() public Job()
{ {
Id = Guid.NewGuid(); Id = Guid.NewGuid();
CreatedAt = DateTime.UtcNow; CreatedAt = DateTime.UtcNow;
Status = BacktestJobStatus.Pending; Status = JobStatus.Pending;
ProgressPercentage = 0; ProgressPercentage = 0;
} }
@@ -39,7 +39,7 @@ public class BacktestJob
/// Current status of the job /// Current status of the job
/// </summary> /// </summary>
[Required] [Required]
public BacktestJobStatus Status { get; set; } public JobStatus Status { get; set; }
/// <summary> /// <summary>
/// Priority of the job (higher = more important) /// Priority of the job (higher = more important)
@@ -122,12 +122,37 @@ public class BacktestJob
/// Optional genetic request ID if this is a genetic backtest job /// Optional genetic request ID if this is a genetic backtest job
/// </summary> /// </summary>
public string? GeneticRequestId { get; set; } public string? GeneticRequestId { get; set; }
/// <summary>
/// Number of times this job has been retried
/// </summary>
public int RetryCount { get; set; } = 0;
/// <summary>
/// Maximum number of retry attempts allowed
/// </summary>
public int MaxRetries { get; set; } = 3;
/// <summary>
/// When the job should be retried next (for exponential backoff)
/// </summary>
public DateTime? RetryAfter { get; set; }
/// <summary>
/// Whether the failure is retryable (transient vs permanent)
/// </summary>
public bool IsRetryable { get; set; } = true;
/// <summary>
/// Failure category for better error handling
/// </summary>
public FailureCategory? FailureCategory { get; set; }
} }
/// <summary> /// <summary>
/// Status of a backtest job /// Status of a job
/// </summary> /// </summary>
public enum BacktestJobStatus public enum JobStatus
{ {
/// <summary> /// <summary>
/// Job is pending and waiting to be claimed by a worker /// Job is pending and waiting to be claimed by a worker
@@ -155,3 +180,34 @@ public enum BacktestJobStatus
Cancelled Cancelled
} }
/// <summary>
/// Category of failure for better error handling and retry logic
/// </summary>
public enum FailureCategory
{
/// <summary>
/// Transient failures: network issues, timeouts, temporary service unavailability
/// </summary>
Transient,
/// <summary>
/// Data errors: missing candles, invalid data format
/// </summary>
DataError,
/// <summary>
/// System errors: out of memory, database errors, infrastructure issues
/// </summary>
SystemError,
/// <summary>
/// User errors: invalid input, configuration errors
/// </summary>
UserError,
/// <summary>
/// Unknown or unclassified errors
/// </summary>
Unknown
}

View File

@@ -0,0 +1,122 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Managing.Infrastructure.Databases.Migrations
{
/// <inheritdoc />
public partial class RenameBacktestJobsToJobs : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropForeignKey(
name: "FK_BacktestJobs_Users_UserId",
table: "BacktestJobs");
migrationBuilder.DropPrimaryKey(
name: "PK_BacktestJobs",
table: "BacktestJobs");
migrationBuilder.RenameTable(
name: "BacktestJobs",
newName: "Jobs");
migrationBuilder.AddColumn<int>(
name: "FailureCategory",
table: "Jobs",
type: "integer",
nullable: true);
migrationBuilder.AddColumn<bool>(
name: "IsRetryable",
table: "Jobs",
type: "boolean",
nullable: false,
defaultValue: false);
migrationBuilder.AddColumn<int>(
name: "MaxRetries",
table: "Jobs",
type: "integer",
nullable: false,
defaultValue: 0);
migrationBuilder.AddColumn<DateTime>(
name: "RetryAfter",
table: "Jobs",
type: "timestamp with time zone",
nullable: true);
migrationBuilder.AddColumn<int>(
name: "RetryCount",
table: "Jobs",
type: "integer",
nullable: false,
defaultValue: 0);
migrationBuilder.AddPrimaryKey(
name: "PK_Jobs",
table: "Jobs",
column: "Id");
migrationBuilder.AddForeignKey(
name: "FK_Jobs_Users_UserId",
table: "Jobs",
column: "UserId",
principalTable: "Users",
principalColumn: "Id",
onDelete: ReferentialAction.SetNull);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropForeignKey(
name: "FK_Jobs_Users_UserId",
table: "Jobs");
migrationBuilder.DropPrimaryKey(
name: "PK_Jobs",
table: "Jobs");
migrationBuilder.DropColumn(
name: "FailureCategory",
table: "Jobs");
migrationBuilder.DropColumn(
name: "IsRetryable",
table: "Jobs");
migrationBuilder.DropColumn(
name: "MaxRetries",
table: "Jobs");
migrationBuilder.DropColumn(
name: "RetryAfter",
table: "Jobs");
migrationBuilder.DropColumn(
name: "RetryCount",
table: "Jobs");
migrationBuilder.RenameTable(
name: "Jobs",
newName: "BacktestJobs");
migrationBuilder.AddPrimaryKey(
name: "PK_BacktestJobs",
table: "BacktestJobs",
column: "Id");
migrationBuilder.AddForeignKey(
name: "FK_BacktestJobs_Users_UserId",
table: "BacktestJobs",
column: "UserId",
principalTable: "Users",
principalColumn: "Id",
onDelete: ReferentialAction.SetNull);
}
}
}

View File

@@ -0,0 +1,77 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Managing.Infrastructure.Databases.Migrations
{
/// <inheritdoc />
public partial class RenameJobsTableToLowercase : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropForeignKey(
name: "FK_Jobs_Users_UserId",
table: "Jobs");
migrationBuilder.DropPrimaryKey(
name: "PK_Jobs",
table: "Jobs");
migrationBuilder.EnsureSchema(
name: "public");
migrationBuilder.RenameTable(
name: "Jobs",
newName: "jobs",
newSchema: "public");
migrationBuilder.AddPrimaryKey(
name: "PK_jobs",
schema: "public",
table: "jobs",
column: "Id");
migrationBuilder.AddForeignKey(
name: "FK_jobs_Users_UserId",
schema: "public",
table: "jobs",
column: "UserId",
principalTable: "Users",
principalColumn: "Id",
onDelete: ReferentialAction.SetNull);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropForeignKey(
name: "FK_jobs_Users_UserId",
schema: "public",
table: "jobs");
migrationBuilder.DropPrimaryKey(
name: "PK_jobs",
schema: "public",
table: "jobs");
migrationBuilder.RenameTable(
name: "jobs",
schema: "public",
newName: "Jobs");
migrationBuilder.AddPrimaryKey(
name: "PK_Jobs",
table: "Jobs",
column: "Id");
migrationBuilder.AddForeignKey(
name: "FK_Jobs_Users_UserId",
table: "Jobs",
column: "UserId",
principalTable: "Users",
principalColumn: "Id",
onDelete: ReferentialAction.SetNull);
}
}
}

View File

@@ -734,10 +734,16 @@ namespace Managing.Infrastructure.Databases.Migrations
b.Property<string>("ErrorMessage") b.Property<string>("ErrorMessage")
.HasColumnType("text"); .HasColumnType("text");
b.Property<int?>("FailureCategory")
.HasColumnType("integer");
b.Property<string>("GeneticRequestId") b.Property<string>("GeneticRequestId")
.HasMaxLength(255) .HasMaxLength(255)
.HasColumnType("character varying(255)"); .HasColumnType("character varying(255)");
b.Property<bool>("IsRetryable")
.HasColumnType("boolean");
b.Property<int>("JobType") b.Property<int>("JobType")
.ValueGeneratedOnAdd() .ValueGeneratedOnAdd()
.HasColumnType("integer") .HasColumnType("integer")
@@ -746,6 +752,9 @@ namespace Managing.Infrastructure.Databases.Migrations
b.Property<DateTime?>("LastHeartbeat") b.Property<DateTime?>("LastHeartbeat")
.HasColumnType("timestamp with time zone"); .HasColumnType("timestamp with time zone");
b.Property<int>("MaxRetries")
.HasColumnType("integer");
b.Property<int>("Priority") b.Property<int>("Priority")
.ValueGeneratedOnAdd() .ValueGeneratedOnAdd()
.HasColumnType("integer") .HasColumnType("integer")
@@ -763,6 +772,12 @@ namespace Managing.Infrastructure.Databases.Migrations
b.Property<string>("ResultJson") b.Property<string>("ResultJson")
.HasColumnType("jsonb"); .HasColumnType("jsonb");
b.Property<DateTime?>("RetryAfter")
.HasColumnType("timestamp with time zone");
b.Property<int>("RetryCount")
.HasColumnType("integer");
b.Property<DateTime>("StartDate") b.Property<DateTime>("StartDate")
.HasColumnType("timestamp with time zone"); .HasColumnType("timestamp with time zone");
@@ -792,7 +807,7 @@ namespace Managing.Infrastructure.Databases.Migrations
b.HasIndex("Status", "JobType", "Priority", "CreatedAt") b.HasIndex("Status", "JobType", "Priority", "CreatedAt")
.HasDatabaseName("idx_status_jobtype_priority_created"); .HasDatabaseName("idx_status_jobtype_priority_created");
b.ToTable("BacktestJobs"); b.ToTable("jobs", "public");
}); });
modelBuilder.Entity("Managing.Infrastructure.Databases.PostgreSql.Entities.MoneyManagementEntity", b => modelBuilder.Entity("Managing.Infrastructure.Databases.PostgreSql.Entities.MoneyManagementEntity", b =>

View File

@@ -3,7 +3,7 @@ using System.ComponentModel.DataAnnotations.Schema;
namespace Managing.Infrastructure.Databases.PostgreSql.Entities; namespace Managing.Infrastructure.Databases.PostgreSql.Entities;
[Table("BacktestJobs")] [Table("Jobs")]
public class JobEntity public class JobEntity
{ {
[Key] [Key]
@@ -16,7 +16,7 @@ public class JobEntity
public int UserId { get; set; } public int UserId { get; set; }
[Required] [Required]
public int Status { get; set; } // BacktestJobStatus enum as int public int Status { get; set; } // JobStatus enum as int
[Required] [Required]
public int JobType { get; set; } // JobType enum as int public int JobType { get; set; } // JobType enum as int
@@ -61,6 +61,16 @@ public class JobEntity
[MaxLength(255)] [MaxLength(255)]
public string? GeneticRequestId { get; set; } public string? GeneticRequestId { get; set; }
public int RetryCount { get; set; } = 0;
public int MaxRetries { get; set; } = 3;
public DateTime? RetryAfter { get; set; }
public bool IsRetryable { get; set; } = true;
public int? FailureCategory { get; set; } // FailureCategory enum as int
// Navigation property // Navigation property
public UserEntity? User { get; set; } public UserEntity? User { get; set; }
} }

View File

@@ -7,7 +7,7 @@ using static Managing.Common.Enums;
namespace Managing.Infrastructure.Databases.PostgreSql; namespace Managing.Infrastructure.Databases.PostgreSql;
public class PostgreSqlJobRepository : IBacktestJobRepository public class PostgreSqlJobRepository : IJobRepository
{ {
private readonly ManagingDbContext _context; private readonly ManagingDbContext _context;
private readonly ILogger<PostgreSqlJobRepository> _logger; private readonly ILogger<PostgreSqlJobRepository> _logger;
@@ -20,7 +20,7 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
_logger = logger; _logger = logger;
} }
public async Task<BacktestJob> CreateAsync(BacktestJob job) public async Task<Job> CreateAsync(Job job)
{ {
var entity = MapToEntity(job); var entity = MapToEntity(job);
_context.Jobs.Add(entity); _context.Jobs.Add(entity);
@@ -28,10 +28,9 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
return MapToDomain(entity); return MapToDomain(entity);
} }
public async Task<BacktestJob?> ClaimNextJobAsync(string workerId, JobType? jobType = null) public async Task<Job?> ClaimNextJobAsync(string workerId, JobType? jobType = null)
{ {
// Use execution strategy to support retry with transactions // Use execution strategy to support retry with transactions
// FOR UPDATE SKIP LOCKED ensures only one worker can claim a specific job
var strategy = _context.Database.CreateExecutionStrategy(); var strategy = _context.Database.CreateExecutionStrategy();
return await strategy.ExecuteAsync(async () => return await strategy.ExecuteAsync(async () =>
@@ -42,10 +41,10 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
{ {
// Build SQL query with optional job type filter // Build SQL query with optional job type filter
var sql = @" var sql = @"
SELECT * FROM ""BacktestJobs"" SELECT * FROM ""Jobs""
WHERE ""Status"" = {0}"; WHERE ""Status"" = {0}";
var parameters = new List<object> { (int)BacktestJobStatus.Pending }; var parameters = new List<object> { (int)JobStatus.Pending };
if (jobType.HasValue) if (jobType.HasValue)
{ {
@@ -70,7 +69,7 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
} }
// Update the job status atomically // Update the job status atomically
job.Status = (int)BacktestJobStatus.Running; job.Status = (int)JobStatus.Running;
job.AssignedWorkerId = workerId; job.AssignedWorkerId = workerId;
job.StartedAt = DateTime.UtcNow; job.StartedAt = DateTime.UtcNow;
job.LastHeartbeat = DateTime.UtcNow; job.LastHeartbeat = DateTime.UtcNow;
@@ -89,7 +88,7 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
}); });
} }
public async Task UpdateAsync(BacktestJob job) public async Task UpdateAsync(Job job)
{ {
// Use AsTracking() to enable change tracking since DbContext uses NoTracking by default // Use AsTracking() to enable change tracking since DbContext uses NoTracking by default
var entity = await _context.Jobs var entity = await _context.Jobs
@@ -115,11 +114,16 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
entity.RequestId = job.RequestId; entity.RequestId = job.RequestId;
entity.GeneticRequestId = job.GeneticRequestId; entity.GeneticRequestId = job.GeneticRequestId;
entity.Priority = job.Priority; entity.Priority = job.Priority;
entity.RetryCount = job.RetryCount;
entity.MaxRetries = job.MaxRetries;
entity.RetryAfter = job.RetryAfter;
entity.IsRetryable = job.IsRetryable;
entity.FailureCategory = job.FailureCategory.HasValue ? (int)job.FailureCategory.Value : null;
await _context.SaveChangesAsync(); await _context.SaveChangesAsync();
} }
public async Task<IEnumerable<BacktestJob>> GetByBundleRequestIdAsync(Guid bundleRequestId) public async Task<IEnumerable<Job>> GetByBundleRequestIdAsync(Guid bundleRequestId)
{ {
var entities = await _context.Jobs var entities = await _context.Jobs
.Where(j => j.BundleRequestId == bundleRequestId) .Where(j => j.BundleRequestId == bundleRequestId)
@@ -128,7 +132,7 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
return entities.Select(MapToDomain); return entities.Select(MapToDomain);
} }
public async Task<IEnumerable<BacktestJob>> GetByUserIdAsync(int userId) public async Task<IEnumerable<Job>> GetByUserIdAsync(int userId)
{ {
var entities = await _context.Jobs var entities = await _context.Jobs
.Where(j => j.UserId == userId) .Where(j => j.UserId == userId)
@@ -140,16 +144,16 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
/// <summary> /// <summary>
/// Gets all running jobs assigned to a specific worker /// Gets all running jobs assigned to a specific worker
/// </summary> /// </summary>
public async Task<IEnumerable<BacktestJob>> GetRunningJobsByWorkerIdAsync(string workerId) public async Task<IEnumerable<Job>> GetRunningJobsByWorkerIdAsync(string workerId)
{ {
var entities = await _context.Jobs var entities = await _context.Jobs
.Where(j => j.AssignedWorkerId == workerId && j.Status == (int)BacktestJobStatus.Running) .Where(j => j.AssignedWorkerId == workerId && j.Status == (int)JobStatus.Running)
.ToListAsync(); .ToListAsync();
return entities.Select(MapToDomain); return entities.Select(MapToDomain);
} }
public async Task<IEnumerable<BacktestJob>> GetByGeneticRequestIdAsync(string geneticRequestId) public async Task<IEnumerable<Job>> GetByGeneticRequestIdAsync(string geneticRequestId)
{ {
var entities = await _context.Jobs var entities = await _context.Jobs
.Where(j => j.GeneticRequestId == geneticRequestId) .Where(j => j.GeneticRequestId == geneticRequestId)
@@ -158,12 +162,12 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
return entities.Select(MapToDomain); return entities.Select(MapToDomain);
} }
public async Task<(IEnumerable<BacktestJob> Jobs, int TotalCount)> GetPaginatedAsync( public async Task<(IEnumerable<Job> Jobs, int TotalCount)> GetPaginatedAsync(
int page, int page,
int pageSize, int pageSize,
string sortBy = "CreatedAt", string sortBy = "CreatedAt",
string sortOrder = "desc", string sortOrder = "desc",
BacktestJobStatus? status = null, JobStatus? status = null,
JobType? jobType = null, JobType? jobType = null,
int? userId = null, int? userId = null,
string? workerId = null, string? workerId = null,
@@ -235,7 +239,7 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
return (jobs, totalCount); return (jobs, totalCount);
} }
public async Task<BacktestJob?> GetByIdAsync(Guid jobId) public async Task<Job?> GetByIdAsync(Guid jobId)
{ {
var entity = await _context.Jobs var entity = await _context.Jobs
.FirstOrDefaultAsync(j => j.Id == jobId); .FirstOrDefaultAsync(j => j.Id == jobId);
@@ -243,12 +247,12 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
return entity != null ? MapToDomain(entity) : null; return entity != null ? MapToDomain(entity) : null;
} }
public async Task<IEnumerable<BacktestJob>> GetStaleJobsAsync(int timeoutMinutes = 5) public async Task<IEnumerable<Job>> GetStaleJobsAsync(int timeoutMinutes = 5)
{ {
var timeoutThreshold = DateTime.UtcNow.AddMinutes(-timeoutMinutes); var timeoutThreshold = DateTime.UtcNow.AddMinutes(-timeoutMinutes);
var entities = await _context.Jobs var entities = await _context.Jobs
.Where(j => j.Status == (int)BacktestJobStatus.Running && .Where(j => j.Status == (int)JobStatus.Running &&
(j.LastHeartbeat == null || j.LastHeartbeat < timeoutThreshold)) (j.LastHeartbeat == null || j.LastHeartbeat < timeoutThreshold))
.ToListAsync(); .ToListAsync();
@@ -262,13 +266,13 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
// Use AsTracking() to enable change tracking since DbContext uses NoTracking by default // Use AsTracking() to enable change tracking since DbContext uses NoTracking by default
var staleJobs = await _context.Jobs var staleJobs = await _context.Jobs
.AsTracking() .AsTracking()
.Where(j => j.Status == (int)BacktestJobStatus.Running && .Where(j => j.Status == (int)JobStatus.Running &&
(j.LastHeartbeat == null || j.LastHeartbeat < timeoutThreshold)) (j.LastHeartbeat == null || j.LastHeartbeat < timeoutThreshold))
.ToListAsync(); .ToListAsync();
foreach (var job in staleJobs) foreach (var job in staleJobs)
{ {
job.Status = (int)BacktestJobStatus.Pending; job.Status = (int)JobStatus.Pending;
job.AssignedWorkerId = null; job.AssignedWorkerId = null;
job.LastHeartbeat = null; job.LastHeartbeat = null;
} }
@@ -299,7 +303,7 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
// Query 1: Status summary // Query 1: Status summary
var statusSummarySql = @" var statusSummarySql = @"
SELECT ""Status"", COUNT(*) as Count SELECT ""Status"", COUNT(*) as Count
FROM ""BacktestJobs"" FROM ""Jobs""
GROUP BY ""Status"" GROUP BY ""Status""
ORDER BY ""Status"""; ORDER BY ""Status""";
@@ -322,7 +326,7 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
// Query 2: Job type summary // Query 2: Job type summary
var jobTypeSummarySql = @" var jobTypeSummarySql = @"
SELECT ""JobType"", COUNT(*) as Count SELECT ""JobType"", COUNT(*) as Count
FROM ""BacktestJobs"" FROM ""Jobs""
GROUP BY ""JobType"" GROUP BY ""JobType""
ORDER BY ""JobType"""; ORDER BY ""JobType""";
@@ -345,7 +349,7 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
// Query 3: Status + Job type summary // Query 3: Status + Job type summary
var statusTypeSummarySql = @" var statusTypeSummarySql = @"
SELECT ""Status"", ""JobType"", COUNT(*) as Count SELECT ""Status"", ""JobType"", COUNT(*) as Count
FROM ""BacktestJobs"" FROM ""Jobs""
GROUP BY ""Status"", ""JobType"" GROUP BY ""Status"", ""JobType""
ORDER BY ""Status"", ""JobType"""; ORDER BY ""Status"", ""JobType""";
@@ -369,7 +373,7 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
// Query 4: Total count // Query 4: Total count
var totalCountSql = @" var totalCountSql = @"
SELECT COUNT(*) as Count SELECT COUNT(*) as Count
FROM ""BacktestJobs"""; FROM ""Jobs""";
using (var command = connection.CreateCommand()) using (var command = connection.CreateCommand())
{ {
@@ -382,7 +386,7 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
{ {
StatusCounts = statusCounts.Select(s => new JobStatusCount StatusCounts = statusCounts.Select(s => new JobStatusCount
{ {
Status = (BacktestJobStatus)s.Status, Status = (JobStatus)s.Status,
Count = s.Count Count = s.Count
}).ToList(), }).ToList(),
JobTypeCounts = jobTypeCounts.Select(j => new JobTypeCount JobTypeCounts = jobTypeCounts.Select(j => new JobTypeCount
@@ -392,7 +396,7 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
}).ToList(), }).ToList(),
StatusTypeCounts = statusTypeCounts.Select(st => new JobStatusTypeCount StatusTypeCounts = statusTypeCounts.Select(st => new JobStatusTypeCount
{ {
Status = (BacktestJobStatus)st.Status, Status = (JobStatus)st.Status,
JobType = (JobType)st.JobType, JobType = (JobType)st.JobType,
Count = st.Count Count = st.Count
}).ToList(), }).ToList(),
@@ -430,7 +434,7 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
public int Count { get; set; } public int Count { get; set; }
} }
private static JobEntity MapToEntity(BacktestJob job) private static JobEntity MapToEntity(Job job)
{ {
return new JobEntity return new JobEntity
{ {
@@ -452,18 +456,23 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
ResultJson = job.ResultJson, ResultJson = job.ResultJson,
ErrorMessage = job.ErrorMessage, ErrorMessage = job.ErrorMessage,
RequestId = job.RequestId, RequestId = job.RequestId,
GeneticRequestId = job.GeneticRequestId GeneticRequestId = job.GeneticRequestId,
RetryCount = job.RetryCount,
MaxRetries = job.MaxRetries,
RetryAfter = job.RetryAfter,
IsRetryable = job.IsRetryable,
FailureCategory = job.FailureCategory.HasValue ? (int)job.FailureCategory.Value : null
}; };
} }
private static BacktestJob MapToDomain(JobEntity entity) private static Job MapToDomain(JobEntity entity)
{ {
return new BacktestJob return new Job
{ {
Id = entity.Id, Id = entity.Id,
BundleRequestId = entity.BundleRequestId, BundleRequestId = entity.BundleRequestId,
UserId = entity.UserId, UserId = entity.UserId,
Status = (BacktestJobStatus)entity.Status, Status = (JobStatus)entity.Status,
JobType = (JobType)entity.JobType, JobType = (JobType)entity.JobType,
Priority = entity.Priority, Priority = entity.Priority,
ConfigJson = entity.ConfigJson, ConfigJson = entity.ConfigJson,
@@ -478,7 +487,12 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
ResultJson = entity.ResultJson, ResultJson = entity.ResultJson,
ErrorMessage = entity.ErrorMessage, ErrorMessage = entity.ErrorMessage,
RequestId = entity.RequestId, RequestId = entity.RequestId,
GeneticRequestId = entity.GeneticRequestId GeneticRequestId = entity.GeneticRequestId,
RetryCount = entity.RetryCount,
MaxRetries = entity.MaxRetries,
RetryAfter = entity.RetryAfter,
IsRetryable = entity.IsRetryable,
FailureCategory = entity.FailureCategory.HasValue ? (FailureCategory)entity.FailureCategory.Value : null
}; };
} }
} }

View File

@@ -19,6 +19,7 @@ const JobsSettings: React.FC = () => {
const [workerIdFilter, setWorkerIdFilter] = useState<string>('') const [workerIdFilter, setWorkerIdFilter] = useState<string>('')
const [bundleRequestIdFilter, setBundleRequestIdFilter] = useState<string>('') const [bundleRequestIdFilter, setBundleRequestIdFilter] = useState<string>('')
const [filtersOpen, setFiltersOpen] = useState<boolean>(false) const [filtersOpen, setFiltersOpen] = useState<boolean>(false)
const [showTable, setShowTable] = useState<boolean>(false)
const jobClient = new JobClient({}, apiUrl) const jobClient = new JobClient({}, apiUrl)
@@ -56,9 +57,10 @@ const JobsSettings: React.FC = () => {
bundleRequestIdFilter || null bundleRequestIdFilter || null
) )
}, },
enabled: showTable, // Only fetch when table is shown
staleTime: 10000, // 10 seconds staleTime: 10000, // 10 seconds
gcTime: 5 * 60 * 1000, gcTime: 5 * 60 * 1000,
refetchInterval: 5000, // Auto-refresh every 5 seconds refetchInterval: showTable ? 5000 : false, // Auto-refresh only when table is shown
}) })
const jobs = jobsData?.jobs || [] const jobs = jobsData?.jobs || []
@@ -316,6 +318,49 @@ const JobsSettings: React.FC = () => {
)} )}
</div> </div>
{/* Load Table Button */}
{!showTable && (
<div className="card bg-base-100 shadow-md mb-4">
<div className="card-body">
<div className="flex items-center justify-between">
<div>
<h3 className="card-title text-lg">Jobs List</h3>
<p className="text-sm text-base-content/70">Click the button below to load and view the jobs table</p>
</div>
<button
className="btn btn-primary"
onClick={() => setShowTable(true)}
>
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" strokeWidth="1.5" stroke="currentColor" className="w-5 h-5">
<path strokeLinecap="round" strokeLinejoin="round" d="M3.75 3v11.25A2.25 2.25 0 006 16.5h2.25M3.75 3h-1.5m1.5 0h16.5m0 0h1.5m-1.5 0v11.25A2.25 2.25 0 0118 16.5h-2.25m-7.5 0h7.5m-7.5 0l-1 3m8.5-3l1 3m0 0l.5 1.5m-.5-1.5h-9.5m0 0l-.5 1.5M9 11.25v1.5M12 9v3.75m3-3.75v3.75m-9 .75h12.75a2.25 2.25 0 002.25-2.25V6.75a2.25 2.25 0 00-2.25-2.25H6.75A2.25 2.25 0 004.5 6.75v7.5a2.25 2.25 0 002.25 2.25z" />
</svg>
Load Jobs Table
</button>
</div>
</div>
</div>
)}
{showTable && (
<>
{/* Hide Table Button */}
<div className="card bg-base-100 shadow-md mb-4">
<div className="card-body py-3">
<div className="flex items-center justify-between">
<h3 className="card-title text-lg">Jobs List</h3>
<button
className="btn btn-ghost btn-sm"
onClick={() => setShowTable(false)}
>
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" strokeWidth="1.5" stroke="currentColor" className="w-5 h-5">
<path strokeLinecap="round" strokeLinejoin="round" d="M6 18L18 6M6 6l12 12" />
</svg>
Hide Table
</button>
</div>
</div>
</div>
{filtersOpen && ( {filtersOpen && (
<div className="card bg-base-200 mb-4"> <div className="card bg-base-200 mb-4">
<div className="card-body"> <div className="card-body">
@@ -434,8 +479,11 @@ const JobsSettings: React.FC = () => {
onPageChange={handlePageChange} onPageChange={handlePageChange}
onSortChange={handleSortChange} onSortChange={handleSortChange}
/> />
</>
)}
{/* Bottom Menu Bar */} {/* Bottom Menu Bar */}
{showTable && (
<BottomMenuBar> <BottomMenuBar>
<li> <li>
<a <a
@@ -502,6 +550,7 @@ const JobsSettings: React.FC = () => {
</a> </a>
</li> </li>
</BottomMenuBar> </BottomMenuBar>
)}
</div> </div>
) )
} }

View File

@@ -265,7 +265,7 @@ const BundleRequestModal: React.FC<BundleRequestModalProps> = ({
const successMessage = asTemplate const successMessage = asTemplate
? 'Template saved successfully!' ? 'Template saved successfully!'
: 'Bundle backtest request created successfully!'; : 'Bundle backtest request created successfully!';
new Toast(successMessage, true); new Toast(successMessage, false);
onClose(); onClose();
} catch (error) { } catch (error) {
const errorMessage = asTemplate const errorMessage = asTemplate

View File

@@ -12,6 +12,8 @@ var host = Host.CreateDefaultBuilder(args)
config.SetBasePath(AppContext.BaseDirectory); config.SetBasePath(AppContext.BaseDirectory);
config.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) config.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
.AddJsonFile($"appsettings.{hostingContext.HostingEnvironment.EnvironmentName}.json", optional: true) .AddJsonFile($"appsettings.{hostingContext.HostingEnvironment.EnvironmentName}.json", optional: true)
.AddJsonFile("appsettings.SandboxLocal.json", optional: true, reloadOnChange: true)
.AddJsonFile("appsettings.ProductionLocal.json", optional: true, reloadOnChange: true)
.AddEnvironmentVariables() .AddEnvironmentVariables()
.AddUserSecrets<Program>(); .AddUserSecrets<Program>();
}) })
@@ -50,6 +52,7 @@ var host = Host.CreateDefaultBuilder(args)
{ {
options.EnableDetailedErrors(); options.EnableDetailedErrors();
options.EnableSensitiveDataLogging(); options.EnableSensitiveDataLogging();
options.LogTo(Console.WriteLine, LogLevel.Information); // Enable SQL logging to debug table name issues
} }
options.UseQueryTrackingBehavior(QueryTrackingBehavior.NoTracking); options.UseQueryTrackingBehavior(QueryTrackingBehavior.NoTracking);
@@ -63,21 +66,47 @@ var host = Host.CreateDefaultBuilder(args)
services.Configure<BacktestComputeWorkerOptions>( services.Configure<BacktestComputeWorkerOptions>(
configuration.GetSection(BacktestComputeWorkerOptions.SectionName)); configuration.GetSection(BacktestComputeWorkerOptions.SectionName));
// Override WorkerId from environment variable if provided // Get task slot from CapRover ({{.Task.Slot}}) or environment variable
// This identifies which instance of the worker is running
var taskSlot = Environment.GetEnvironmentVariable("TASK_SLOT") ??
Environment.GetEnvironmentVariable("CAPROVER_TASK_SLOT") ??
"0";
// Override WorkerId from environment variable if provided, otherwise use task slot
var workerId = Environment.GetEnvironmentVariable("WORKER_ID") ?? var workerId = Environment.GetEnvironmentVariable("WORKER_ID") ??
configuration["BacktestComputeWorker:WorkerId"] ?? configuration["BacktestComputeWorker:WorkerId"] ??
Environment.MachineName; $"{Environment.MachineName}-{taskSlot}";
services.Configure<BacktestComputeWorkerOptions>(options => services.Configure<BacktestComputeWorkerOptions>(options =>
{ {
options.WorkerId = workerId; options.WorkerId = workerId;
}); });
// Register the compute worker if enabled // Configure GeneticComputeWorker options
var isWorkerEnabled = configuration.GetValue<bool>("WorkerBacktestCompute", false); services.Configure<GeneticComputeWorkerOptions>(
if (isWorkerEnabled) configuration.GetSection(GeneticComputeWorkerOptions.SectionName));
// Override Genetic WorkerId from environment variable if provided, otherwise use task slot
var geneticWorkerId = Environment.GetEnvironmentVariable("GENETIC_WORKER_ID") ??
configuration["GeneticComputeWorker:WorkerId"] ??
$"{Environment.MachineName}-genetic-{taskSlot}";
services.Configure<GeneticComputeWorkerOptions>(options =>
{
options.WorkerId = geneticWorkerId;
});
// Register the backtest compute worker if enabled
var isBacktestWorkerEnabled = configuration.GetValue<bool>("WorkerBacktestCompute", false);
if (isBacktestWorkerEnabled)
{ {
services.AddHostedService<BacktestComputeWorker>(); services.AddHostedService<BacktestComputeWorker>();
} }
// Register the genetic compute worker if enabled
var isGeneticWorkerEnabled = configuration.GetValue<bool>("WorkerGeneticCompute", false);
if (isGeneticWorkerEnabled)
{
services.AddHostedService<GeneticComputeWorker>();
}
}) })
.ConfigureLogging((hostingContext, logging) => .ConfigureLogging((hostingContext, logging) =>
{ {
@@ -89,18 +118,41 @@ var host = Host.CreateDefaultBuilder(args)
// Log worker status // Log worker status
var logger = host.Services.GetRequiredService<ILogger<Program>>(); var logger = host.Services.GetRequiredService<ILogger<Program>>();
var isWorkerEnabled = host.Services.GetRequiredService<IConfiguration>().GetValue<bool>("WorkerBacktestCompute", false); var config = host.Services.GetRequiredService<IConfiguration>();
if (isWorkerEnabled) var isBacktestWorkerEnabled = config.GetValue<bool>("WorkerBacktestCompute", false);
var isGeneticWorkerEnabled = config.GetValue<bool>("WorkerGeneticCompute", false);
if (isBacktestWorkerEnabled)
{ {
var taskSlot = Environment.GetEnvironmentVariable("TASK_SLOT") ??
Environment.GetEnvironmentVariable("CAPROVER_TASK_SLOT") ??
"0";
var backtestWorkerId = Environment.GetEnvironmentVariable("WORKER_ID") ??
config["BacktestComputeWorker:WorkerId"] ??
$"{Environment.MachineName}-{taskSlot}";
logger.LogInformation("BacktestComputeWorker is enabled and will be started."); logger.LogInformation("BacktestComputeWorker is enabled and will be started.");
logger.LogInformation("Worker ID: {WorkerId}", Environment.GetEnvironmentVariable("WORKER_ID") ?? logger.LogInformation("Backtest Worker ID: {WorkerId} (Task Slot: {TaskSlot})", backtestWorkerId, taskSlot);
host.Services.GetRequiredService<IConfiguration>()["BacktestComputeWorker:WorkerId"] ??
Environment.MachineName);
} }
else else
{ {
logger.LogWarning("BacktestComputeWorker is disabled via configuration. No jobs will be processed."); logger.LogWarning("BacktestComputeWorker is disabled via configuration. No backtest jobs will be processed.");
}
if (isGeneticWorkerEnabled)
{
var taskSlot = Environment.GetEnvironmentVariable("TASK_SLOT") ??
Environment.GetEnvironmentVariable("CAPROVER_TASK_SLOT") ??
"0";
var geneticWorkerId = Environment.GetEnvironmentVariable("GENETIC_WORKER_ID") ??
config["GeneticComputeWorker:WorkerId"] ??
$"{Environment.MachineName}-genetic-{taskSlot}";
logger.LogInformation("GeneticComputeWorker is enabled and will be started.");
logger.LogInformation("Genetic Worker ID: {WorkerId} (Task Slot: {TaskSlot})", geneticWorkerId, taskSlot);
}
else
{
logger.LogWarning("GeneticComputeWorker is disabled via configuration. No genetic jobs will be processed.");
} }
try try

View File

@@ -7,12 +7,18 @@
}, },
"WorkerBacktestCompute": true, "WorkerBacktestCompute": true,
"BacktestComputeWorker": { "BacktestComputeWorker": {
"WorkerId": "local-worker-1",
"MaxConcurrentBacktests": 6, "MaxConcurrentBacktests": 6,
"JobPollIntervalSeconds": 5, "JobPollIntervalSeconds": 5,
"HeartbeatIntervalSeconds": 30, "HeartbeatIntervalSeconds": 30,
"StaleJobTimeoutMinutes": 5 "StaleJobTimeoutMinutes": 5
}, },
"WorkerGeneticCompute": true,
"GeneticComputeWorker": {
"MaxConcurrentGenetics": 2,
"JobPollIntervalSeconds": 5,
"HeartbeatIntervalSeconds": 30,
"StaleJobTimeoutMinutes": 10
},
"PostgreSql": { "PostgreSql": {
"ConnectionString": "Host=localhost;Port=5432;Database=managing;Username=postgres;Password=postgres" "ConnectionString": "Host=localhost;Port=5432;Database=managing;Username=postgres;Password=postgres"
}, },

View File

@@ -0,0 +1,45 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft.Hosting.Lifetime": "Information"
}
},
"WorkerBacktestCompute": true,
"BacktestComputeWorker": {
"MaxConcurrentBacktests": 6,
"JobPollIntervalSeconds": 5,
"HeartbeatIntervalSeconds": 30,
"StaleJobTimeoutMinutes": 5
},
"WorkerGeneticCompute": true,
"GeneticComputeWorker": {
"MaxConcurrentGenetics": 2,
"JobPollIntervalSeconds": 5,
"HeartbeatIntervalSeconds": 30,
"StaleJobTimeoutMinutes": 10
},
"PostgreSql": {
"ConnectionString": "Host=kaigen-db.kaigen.managing.live;Port=5432;Database=managing;Username=postgres;Password=2ab5423dcca4aa2d"
},
"InfluxDb": {
"Url": "https://influx-db.kaigen.managing.live",
"Organization": "managing-org",
"Token": "ROvQoZ1Dg5jiKDFxB0saEGqHC3rsLkUNlPL6_AFbOcpNjMieIv8v58yA4v5tFU9sX9LLvXEToPvUrxqQEMaWDw=="
},
"Sentry": {
"Dsn": "https://fe12add48c56419bbdfa86227c188e7a@glitch.kai.managing.live/1"
},
"N8n": {
"WebhookUrl": "https://n8n.kai.managing.live/webhook/fa9308b6-983b-42ec-b085-71599d655951",
"IndicatorRequestWebhookUrl": "https://n8n.kai.managing.live/webhook/3aa07b66-1e64-46a7-8618-af300914cb11",
"Username": "managing-api",
"Password": "T259836*PdiV2@%!eR%Qf4"
},
"Kaigen": {
"BaseUrl": "https://kaigen-back-kaigen-stage.up.railway.app",
"DebitEndpoint": "/api/credits/debit",
"RefundEndpoint": "/api/credits/refund"
}
}

View File

@@ -0,0 +1,45 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft.Hosting.Lifetime": "Information"
}
},
"WorkerBacktestCompute": true,
"BacktestComputeWorker": {
"MaxConcurrentBacktests": 6,
"JobPollIntervalSeconds": 5,
"HeartbeatIntervalSeconds": 30,
"StaleJobTimeoutMinutes": 5
},
"WorkerGeneticCompute": true,
"GeneticComputeWorker": {
"MaxConcurrentGenetics": 2,
"JobPollIntervalSeconds": 5,
"HeartbeatIntervalSeconds": 30,
"StaleJobTimeoutMinutes": 10
},
"PostgreSql": {
"ConnectionString": "Host=managing-postgre.apps.managing.live;Port=5432;Database=managing;Username=postgres;Password=29032b13a5bc4d37"
},
"InfluxDb": {
"Url": "https://influx-db.apps.managing.live",
"Organization": "managing-org",
"Token": "zODh8Hn8sN5VwpVJH0HAwDpCJPE4oB5IUg8L4Q0T67KM1Rta6PoM0nATUzf1ddkyWx_VledooZXfFIddahbL9Q=="
},
"Sentry": {
"Dsn": "https://fe12add48c56419bbdfa86227c188e7a@glitch.kai.managing.live/1"
},
"N8n": {
"WebhookUrl": "https://n8n.kai.managing.live/webhook/fa9308b6-983b-42ec-b085-71599d655951",
"IndicatorRequestWebhookUrl": "https://n8n.kai.managing.live/webhook/3aa07b66-1e64-46a7-8618-af300914cb11",
"Username": "managing-api",
"Password": "T259836*PdiV2@%!eR%Qf4"
},
"Kaigen": {
"BaseUrl": "https://kaigen-back-kaigen-stage.up.railway.app",
"DebitEndpoint": "/api/credits/debit",
"RefundEndpoint": "/api/credits/refund"
}
}

View File

@@ -7,12 +7,18 @@
}, },
"WorkerBacktestCompute": true, "WorkerBacktestCompute": true,
"BacktestComputeWorker": { "BacktestComputeWorker": {
"WorkerId": "worker-1",
"MaxConcurrentBacktests": 6, "MaxConcurrentBacktests": 6,
"JobPollIntervalSeconds": 5, "JobPollIntervalSeconds": 5,
"HeartbeatIntervalSeconds": 30, "HeartbeatIntervalSeconds": 30,
"StaleJobTimeoutMinutes": 5 "StaleJobTimeoutMinutes": 5
}, },
"WorkerGeneticCompute": true,
"GeneticComputeWorker": {
"MaxConcurrentGenetics": 2,
"JobPollIntervalSeconds": 5,
"HeartbeatIntervalSeconds": 30,
"StaleJobTimeoutMinutes": 10
},
"PostgreSql": { "PostgreSql": {
"ConnectionString": "Host=managing-postgre.apps.managing.live;Port=5432;Database=managing;Username=postgres;Password=29032b13a5bc4d37" "ConnectionString": "Host=managing-postgre.apps.managing.live;Port=5432;Database=managing;Username=postgres;Password=29032b13a5bc4d37"
}, },