Oda/managing-apps
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add genetic backtest to worker

Browse Source
This commit is contained in:
cryptooda
2025-11-09 03:32:08 +07:00
parent 7dba29c66f
commit 7e08e63dd1
30 changed files with 5056 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1720
src/Managing.Infrastructure.Database/Migrations/20251108195534_RenameBacktestJobsToJobs.Designer.cs generated Normal file
View File

File diff suppressed because it is too large Load Diff

122
src/Managing.Infrastructure.Database/Migrations/20251108195534_RenameBacktestJobsToJobs.cs Normal file
View File

@@ -0,0 +1,122 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Managing.Infrastructure.Databases.Migrations
{
/// <inheritdoc />
public partial class RenameBacktestJobsToJobs : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropForeignKey(
name: "FK_BacktestJobs_Users_UserId",
table: "BacktestJobs");
migrationBuilder.DropPrimaryKey(
name: "PK_BacktestJobs",
table: "BacktestJobs");
migrationBuilder.RenameTable(
name: "BacktestJobs",
newName: "Jobs");
migrationBuilder.AddColumn<int>(
name: "FailureCategory",
table: "Jobs",
type: "integer",
nullable: true);
migrationBuilder.AddColumn<bool>(
name: "IsRetryable",
table: "Jobs",
type: "boolean",
nullable: false,
defaultValue: false);
migrationBuilder.AddColumn<int>(
name: "MaxRetries",
table: "Jobs",
type: "integer",
nullable: false,
defaultValue: 0);
migrationBuilder.AddColumn<DateTime>(
name: "RetryAfter",
table: "Jobs",
type: "timestamp with time zone",
nullable: true);
migrationBuilder.AddColumn<int>(
name: "RetryCount",
table: "Jobs",
type: "integer",
nullable: false,
defaultValue: 0);
migrationBuilder.AddPrimaryKey(
name: "PK_Jobs",
table: "Jobs",
column: "Id");
migrationBuilder.AddForeignKey(
name: "FK_Jobs_Users_UserId",
table: "Jobs",
column: "UserId",
principalTable: "Users",
principalColumn: "Id",
onDelete: ReferentialAction.SetNull);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropForeignKey(
name: "FK_Jobs_Users_UserId",
table: "Jobs");
migrationBuilder.DropPrimaryKey(
name: "PK_Jobs",
table: "Jobs");
migrationBuilder.DropColumn(
name: "FailureCategory",
table: "Jobs");
migrationBuilder.DropColumn(
name: "IsRetryable",
table: "Jobs");
migrationBuilder.DropColumn(
name: "MaxRetries",
table: "Jobs");
migrationBuilder.DropColumn(
name: "RetryAfter",
table: "Jobs");
migrationBuilder.DropColumn(
name: "RetryCount",
table: "Jobs");
migrationBuilder.RenameTable(
name: "Jobs",
newName: "BacktestJobs");
migrationBuilder.AddPrimaryKey(
name: "PK_BacktestJobs",
table: "BacktestJobs",
column: "Id");
migrationBuilder.AddForeignKey(
name: "FK_BacktestJobs_Users_UserId",
table: "BacktestJobs",
column: "UserId",
principalTable: "Users",
principalColumn: "Id",
onDelete: ReferentialAction.SetNull);
}
}
}

1720
src/Managing.Infrastructure.Database/Migrations/20251108202235_RenameJobsTableToLowercase.Designer.cs generated Normal file
View File

File diff suppressed because it is too large Load Diff

77
src/Managing.Infrastructure.Database/Migrations/20251108202235_RenameJobsTableToLowercase.cs Normal file
View File

@@ -0,0 +1,77 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Managing.Infrastructure.Databases.Migrations
{
/// <inheritdoc />
public partial class RenameJobsTableToLowercase : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropForeignKey(
name: "FK_Jobs_Users_UserId",
table: "Jobs");
migrationBuilder.DropPrimaryKey(
name: "PK_Jobs",
table: "Jobs");
migrationBuilder.EnsureSchema(
name: "public");
migrationBuilder.RenameTable(
name: "Jobs",
newName: "jobs",
newSchema: "public");
migrationBuilder.AddPrimaryKey(
name: "PK_jobs",
schema: "public",
table: "jobs",
column: "Id");
migrationBuilder.AddForeignKey(
name: "FK_jobs_Users_UserId",
schema: "public",
table: "jobs",
column: "UserId",
principalTable: "Users",
principalColumn: "Id",
onDelete: ReferentialAction.SetNull);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropForeignKey(
name: "FK_jobs_Users_UserId",
schema: "public",
table: "jobs");
migrationBuilder.DropPrimaryKey(
name: "PK_jobs",
schema: "public",
table: "jobs");
migrationBuilder.RenameTable(
name: "jobs",
schema: "public",
newName: "Jobs");
migrationBuilder.AddPrimaryKey(
name: "PK_Jobs",
table: "Jobs",
column: "Id");
migrationBuilder.AddForeignKey(
name: "FK_Jobs_Users_UserId",
table: "Jobs",
column: "UserId",
principalTable: "Users",
principalColumn: "Id",
onDelete: ReferentialAction.SetNull);
}
}
}

17
src/Managing.Infrastructure.Database/Migrations/ManagingDbContextModelSnapshot.cs
View File

@@ -734,10 +734,16 @@ namespace Managing.Infrastructure.Databases.Migrations
b.Property<string>("ErrorMessage")
.HasColumnType("text");
b.Property<int?>("FailureCategory")
.HasColumnType("integer");
b.Property<string>("GeneticRequestId")
.HasMaxLength(255)
.HasColumnType("character varying(255)");
b.Property<bool>("IsRetryable")
.HasColumnType("boolean");
b.Property<int>("JobType")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
@@ -746,6 +752,9 @@ namespace Managing.Infrastructure.Databases.Migrations
b.Property<DateTime?>("LastHeartbeat")
.HasColumnType("timestamp with time zone");
b.Property<int>("MaxRetries")
.HasColumnType("integer");
b.Property<int>("Priority")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
@@ -763,6 +772,12 @@ namespace Managing.Infrastructure.Databases.Migrations
b.Property<string>("ResultJson")
.HasColumnType("jsonb");
b.Property<DateTime?>("RetryAfter")
.HasColumnType("timestamp with time zone");
b.Property<int>("RetryCount")
.HasColumnType("integer");
b.Property<DateTime>("StartDate")
.HasColumnType("timestamp with time zone");
@@ -792,7 +807,7 @@ namespace Managing.Infrastructure.Databases.Migrations
b.HasIndex("Status", "JobType", "Priority", "CreatedAt")
.HasDatabaseName("idx_status_jobtype_priority_created");
b.ToTable("BacktestJobs");
b.ToTable("jobs", "public");
});
modelBuilder.Entity("Managing.Infrastructure.Databases.PostgreSql.Entities.MoneyManagementEntity", b =>

14
src/Managing.Infrastructure.Database/PostgreSql/Entities/JobEntity.cs
View File

@@ -3,7 +3,7 @@ using System.ComponentModel.DataAnnotations.Schema;
namespace Managing.Infrastructure.Databases.PostgreSql.Entities;
[Table("BacktestJobs")]
[Table("Jobs")]
public class JobEntity
{
[Key]
@@ -16,7 +16,7 @@ public class JobEntity
public int UserId { get; set; }
[Required]
public int Status { get; set; } // BacktestJobStatus enum as int
public int Status { get; set; } // JobStatus enum as int
[Required]
public int JobType { get; set; } // JobType enum as int
@@ -61,6 +61,16 @@ public class JobEntity
[MaxLength(255)]
public string? GeneticRequestId { get; set; }
public int RetryCount { get; set; } = 0;
public int MaxRetries { get; set; } = 3;
public DateTime? RetryAfter { get; set; }
public bool IsRetryable { get; set; } = true;
public int? FailureCategory { get; set; } // FailureCategory enum as int
// Navigation property
public UserEntity? User { get; set; }
}

78
src/Managing.Infrastructure.Database/PostgreSql/PostgreSqlJobRepository.cs
View File

@@ -7,7 +7,7 @@ using static Managing.Common.Enums;
namespace Managing.Infrastructure.Databases.PostgreSql;
public class PostgreSqlJobRepository : IBacktestJobRepository
public class PostgreSqlJobRepository : IJobRepository
{
private readonly ManagingDbContext _context;
private readonly ILogger<PostgreSqlJobRepository> _logger;
@@ -20,7 +20,7 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
_logger = logger;
}
public async Task<BacktestJob> CreateAsync(BacktestJob job)
public async Task<Job> CreateAsync(Job job)
{
var entity = MapToEntity(job);
_context.Jobs.Add(entity);
@@ -28,10 +28,9 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
return MapToDomain(entity);
}
public async Task<BacktestJob?> ClaimNextJobAsync(string workerId, JobType? jobType = null)
public async Task<Job?> ClaimNextJobAsync(string workerId, JobType? jobType = null)
{
// Use execution strategy to support retry with transactions
// FOR UPDATE SKIP LOCKED ensures only one worker can claim a specific job
var strategy = _context.Database.CreateExecutionStrategy();
return await strategy.ExecuteAsync(async () =>
@@ -42,10 +41,10 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
{
// Build SQL query with optional job type filter
var sql = @"
SELECT * FROM ""BacktestJobs""
SELECT * FROM ""Jobs""
WHERE ""Status"" = {0}";
var parameters = new List<object> { (int)BacktestJobStatus.Pending };
var parameters = new List<object> { (int)JobStatus.Pending };
if (jobType.HasValue)
{
@@ -70,7 +69,7 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
}
// Update the job status atomically
job.Status = (int)BacktestJobStatus.Running;
job.Status = (int)JobStatus.Running;
job.AssignedWorkerId = workerId;
job.StartedAt = DateTime.UtcNow;
job.LastHeartbeat = DateTime.UtcNow;
@@ -89,7 +88,7 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
});
}
public async Task UpdateAsync(BacktestJob job)
public async Task UpdateAsync(Job job)
{
// Use AsTracking() to enable change tracking since DbContext uses NoTracking by default
var entity = await _context.Jobs
@@ -115,11 +114,16 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
entity.RequestId = job.RequestId;
entity.GeneticRequestId = job.GeneticRequestId;
entity.Priority = job.Priority;
entity.RetryCount = job.RetryCount;
entity.MaxRetries = job.MaxRetries;
entity.RetryAfter = job.RetryAfter;
entity.IsRetryable = job.IsRetryable;
entity.FailureCategory = job.FailureCategory.HasValue ? (int)job.FailureCategory.Value : null;
await _context.SaveChangesAsync();
}
public async Task<IEnumerable<BacktestJob>> GetByBundleRequestIdAsync(Guid bundleRequestId)
public async Task<IEnumerable<Job>> GetByBundleRequestIdAsync(Guid bundleRequestId)
{
var entities = await _context.Jobs
.Where(j => j.BundleRequestId == bundleRequestId)
@@ -128,7 +132,7 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
return entities.Select(MapToDomain);
}
public async Task<IEnumerable<BacktestJob>> GetByUserIdAsync(int userId)
public async Task<IEnumerable<Job>> GetByUserIdAsync(int userId)
{
var entities = await _context.Jobs
.Where(j => j.UserId == userId)
@@ -140,16 +144,16 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
/// <summary>
/// Gets all running jobs assigned to a specific worker
/// </summary>
public async Task<IEnumerable<BacktestJob>> GetRunningJobsByWorkerIdAsync(string workerId)
public async Task<IEnumerable<Job>> GetRunningJobsByWorkerIdAsync(string workerId)
{
var entities = await _context.Jobs
.Where(j => j.AssignedWorkerId == workerId && j.Status == (int)BacktestJobStatus.Running)
.Where(j => j.AssignedWorkerId == workerId && j.Status == (int)JobStatus.Running)
.ToListAsync();
return entities.Select(MapToDomain);
}
public async Task<IEnumerable<BacktestJob>> GetByGeneticRequestIdAsync(string geneticRequestId)
public async Task<IEnumerable<Job>> GetByGeneticRequestIdAsync(string geneticRequestId)
{
var entities = await _context.Jobs
.Where(j => j.GeneticRequestId == geneticRequestId)
@@ -158,12 +162,12 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
return entities.Select(MapToDomain);
}
public async Task<(IEnumerable<BacktestJob> Jobs, int TotalCount)> GetPaginatedAsync(
public async Task<(IEnumerable<Job> Jobs, int TotalCount)> GetPaginatedAsync(
int page,
int pageSize,
string sortBy = "CreatedAt",
string sortOrder = "desc",
BacktestJobStatus? status = null,
JobStatus? status = null,
JobType? jobType = null,
int? userId = null,
string? workerId = null,
@@ -235,7 +239,7 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
return (jobs, totalCount);
}
public async Task<BacktestJob?> GetByIdAsync(Guid jobId)
public async Task<Job?> GetByIdAsync(Guid jobId)
{
var entity = await _context.Jobs
.FirstOrDefaultAsync(j => j.Id == jobId);
@@ -243,12 +247,12 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
return entity != null ? MapToDomain(entity) : null;
}
public async Task<IEnumerable<BacktestJob>> GetStaleJobsAsync(int timeoutMinutes = 5)
public async Task<IEnumerable<Job>> GetStaleJobsAsync(int timeoutMinutes = 5)
{
var timeoutThreshold = DateTime.UtcNow.AddMinutes(-timeoutMinutes);
var entities = await _context.Jobs
.Where(j => j.Status == (int)BacktestJobStatus.Running &&
.Where(j => j.Status == (int)JobStatus.Running &&
(j.LastHeartbeat == null || j.LastHeartbeat < timeoutThreshold))
.ToListAsync();
@@ -262,13 +266,13 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
// Use AsTracking() to enable change tracking since DbContext uses NoTracking by default
var staleJobs = await _context.Jobs
.AsTracking()
.Where(j => j.Status == (int)BacktestJobStatus.Running &&
.Where(j => j.Status == (int)JobStatus.Running &&
(j.LastHeartbeat == null || j.LastHeartbeat < timeoutThreshold))
.ToListAsync();
foreach (var job in staleJobs)
{
job.Status = (int)BacktestJobStatus.Pending;
job.Status = (int)JobStatus.Pending;
job.AssignedWorkerId = null;
job.LastHeartbeat = null;
}
@@ -299,7 +303,7 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
// Query 1: Status summary
var statusSummarySql = @"
SELECT ""Status"", COUNT(*) as Count
FROM ""BacktestJobs""
FROM ""Jobs""
GROUP BY ""Status""
ORDER BY ""Status""";
@@ -322,7 +326,7 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
// Query 2: Job type summary
var jobTypeSummarySql = @"
SELECT ""JobType"", COUNT(*) as Count
FROM ""BacktestJobs""
FROM ""Jobs""
GROUP BY ""JobType""
ORDER BY ""JobType""";
@@ -345,7 +349,7 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
// Query 3: Status + Job type summary
var statusTypeSummarySql = @"
SELECT ""Status"", ""JobType"", COUNT(*) as Count
FROM ""BacktestJobs""
FROM ""Jobs""
GROUP BY ""Status"", ""JobType""
ORDER BY ""Status"", ""JobType""";
@@ -369,7 +373,7 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
// Query 4: Total count
var totalCountSql = @"
SELECT COUNT(*) as Count
FROM ""BacktestJobs""";
FROM ""Jobs""";
using (var command = connection.CreateCommand())
{
@@ -382,7 +386,7 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
{
StatusCounts = statusCounts.Select(s => new JobStatusCount
{
Status = (BacktestJobStatus)s.Status,
Status = (JobStatus)s.Status,
Count = s.Count
}).ToList(),
JobTypeCounts = jobTypeCounts.Select(j => new JobTypeCount
@@ -392,7 +396,7 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
}).ToList(),
StatusTypeCounts = statusTypeCounts.Select(st => new JobStatusTypeCount
{
Status = (BacktestJobStatus)st.Status,
Status = (JobStatus)st.Status,
JobType = (JobType)st.JobType,
Count = st.Count
}).ToList(),
@@ -430,7 +434,7 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
public int Count { get; set; }
}
private static JobEntity MapToEntity(BacktestJob job)
private static JobEntity MapToEntity(Job job)
{
return new JobEntity
{
@@ -452,18 +456,23 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
ResultJson = job.ResultJson,
ErrorMessage = job.ErrorMessage,
RequestId = job.RequestId,
GeneticRequestId = job.GeneticRequestId
GeneticRequestId = job.GeneticRequestId,
RetryCount = job.RetryCount,
MaxRetries = job.MaxRetries,
RetryAfter = job.RetryAfter,
IsRetryable = job.IsRetryable,
FailureCategory = job.FailureCategory.HasValue ? (int)job.FailureCategory.Value : null
};
}
private static BacktestJob MapToDomain(JobEntity entity)
private static Job MapToDomain(JobEntity entity)
{
return new BacktestJob
return new Job
{
Id = entity.Id,
BundleRequestId = entity.BundleRequestId,
UserId = entity.UserId,
Status = (BacktestJobStatus)entity.Status,
Status = (JobStatus)entity.Status,
JobType = (JobType)entity.JobType,
Priority = entity.Priority,
ConfigJson = entity.ConfigJson,
@@ -478,7 +487,12 @@ public class PostgreSqlJobRepository : IBacktestJobRepository
ResultJson = entity.ResultJson,
ErrorMessage = entity.ErrorMessage,
RequestId = entity.RequestId,
GeneticRequestId = entity.GeneticRequestId
GeneticRequestId = entity.GeneticRequestId,
RetryCount = entity.RetryCount,
MaxRetries = entity.MaxRetries,
RetryAfter = entity.RetryAfter,
IsRetryable = entity.IsRetryable,
FailureCategory = entity.FailureCategory.HasValue ? (FailureCategory)entity.FailureCategory.Value : null
};
}
}