Oda/managing-apps
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update jobs

Browse Source
This commit is contained in:
cryptooda
2025-11-09 04:48:15 +07:00
parent 7e08e63dd1
commit 747bda2700
20 changed files with 2281 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/Dockerfile-worker-api-dev
View File

@@ -5,7 +5,7 @@ WORKDIR /app
# Use the official Microsoft .NET SDK image to build the code.
FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
WORKDIR /buildapp
COPY ["/src/Managing.Workers.Api/Managing.Workers.Api.csproj", "Managing.Workers.Api/"]
COPY ["/src/Managing.Workers/Managing.Workers.csproj", "Managing.Workers/"]
COPY ["/src/Managing.Bootstrap/Managing.Bootstrap.csproj", "Managing.Bootstrap/"]
COPY ["/src/Managing.Application/Managing.Application.csproj", "Managing.Application/"]
COPY ["/src/Managing.Application.Abstractions/Managing.Application.Abstractions.csproj", "Managing.Application.Abstractions/"]
@@ -17,15 +17,15 @@ COPY ["/src/Managing.Infrastructure.Exchanges/Managing.Infrastructure.Exchanges.
COPY ["/src/Managing.Infrastructure.Messengers/Managing.Infrastructure.Messengers.csproj", "Managing.Infrastructure.Messengers/"]
COPY ["/src/Managing.Infrastructure.Storage/Managing.Infrastructure.Storage.csproj", "Managing.Infrastructure.Storage/"]
COPY ["/src/Managing.Infrastructure.Web3/Managing.Infrastructure.Evm.csproj", "Managing.Infrastructure.Web3/"]
RUN dotnet restore "/buildapp/Managing.Workers.Api/Managing.Workers.Api.csproj"
RUN dotnet restore "/buildapp/Managing.Workers/Managing.Workers.csproj"
COPY . .
WORKDIR "/buildapp/src/Managing.Workers.Api"
RUN dotnet build "Managing.Workers.Api.csproj" -c Release -o /app/build
WORKDIR "/buildapp/src/Managing.Workers"
RUN dotnet build "Managing.Workers.csproj" -c Release -o /app/build
FROM build AS publish
RUN dotnet publish "Managing.Workers.Api.csproj" -c Release -o /app/publish
RUN dotnet publish "Managing.Workers.csproj" -c Release -o /app/publish
FROM base AS final
WORKDIR /app
COPY --from=publish /app/publish .
ENTRYPOINT ["dotnet", "Managing.Workers.Api.dll"]
ENTRYPOINT ["dotnet", "Managing.Workers.dll"]

1720
src/Managing.Infrastructure.Database/Migrations/20251108203747_RenameJobsTableToUppercase.Designer.cs generated Normal file
View File

File diff suppressed because it is too large Load Diff

77
src/Managing.Infrastructure.Database/Migrations/20251108203747_RenameJobsTableToUppercase.cs Normal file
View File

@@ -0,0 +1,77 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Managing.Infrastructure.Databases.Migrations
{
/// <inheritdoc />
public partial class RenameJobsTableToUppercase : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropForeignKey(
name: "FK_jobs_Users_UserId",
schema: "public",
table: "jobs");
migrationBuilder.DropPrimaryKey(
name: "PK_jobs",
schema: "public",
table: "jobs");
migrationBuilder.RenameTable(
name: "jobs",
schema: "public",
newName: "Jobs");
migrationBuilder.AddPrimaryKey(
name: "PK_Jobs",
table: "Jobs",
column: "Id");
migrationBuilder.AddForeignKey(
name: "FK_Jobs_Users_UserId",
table: "Jobs",
column: "UserId",
principalTable: "Users",
principalColumn: "Id",
onDelete: ReferentialAction.SetNull);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropForeignKey(
name: "FK_Jobs_Users_UserId",
table: "Jobs");
migrationBuilder.DropPrimaryKey(
name: "PK_Jobs",
table: "Jobs");
migrationBuilder.EnsureSchema(
name: "public");
migrationBuilder.RenameTable(
name: "Jobs",
newName: "jobs",
newSchema: "public");
migrationBuilder.AddPrimaryKey(
name: "PK_jobs",
schema: "public",
table: "jobs",
column: "Id");
migrationBuilder.AddForeignKey(
name: "FK_jobs_Users_UserId",
schema: "public",
table: "jobs",
column: "UserId",
principalTable: "Users",
principalColumn: "Id",
onDelete: ReferentialAction.SetNull);
}
}
}

2
src/Managing.Infrastructure.Database/Migrations/ManagingDbContextModelSnapshot.cs
View File

@@ -807,7 +807,7 @@ namespace Managing.Infrastructure.Databases.Migrations
b.HasIndex("Status", "JobType", "Priority", "CreatedAt")
.HasDatabaseName("idx_status_jobtype_priority_created");
b.ToTable("jobs", "public");
b.ToTable("Jobs", (string)null);
});
modelBuilder.Entity("Managing.Infrastructure.Databases.PostgreSql.Entities.MoneyManagementEntity", b =>

3
src/Managing.Infrastructure.Database/PostgreSql/ManagingDbContext.cs
View File

@@ -269,6 +269,9 @@ public class ManagingDbContext : DbContext
.WithMany()
.HasForeignKey(e => e.UserId)
.OnDelete(DeleteBehavior.SetNull);
// Explicitly set table name to "Jobs" (uppercase)
entity.ToTable("Jobs");
});
// Configure Scenario entity

80
src/Managing.Infrastructure.Database/PostgreSql/PostgreSqlJobRepository.cs
View File

@@ -2,7 +2,10 @@ using Managing.Application.Abstractions.Repositories;
using Managing.Domain.Backtests;
using Managing.Infrastructure.Databases.PostgreSql.Entities;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Storage;
using Microsoft.Extensions.Logging;
using Npgsql;
using NpgsqlTypes;
using static Managing.Common.Enums;
namespace Managing.Infrastructure.Databases.PostgreSql;
@@ -40,16 +43,26 @@ public class PostgreSqlJobRepository : IJobRepository
try
{
// Build SQL query with optional job type filter
// Use raw ADO.NET to avoid EF Core wrapping the query (which breaks FOR UPDATE SKIP LOCKED)
var sql = @"
SELECT * FROM ""Jobs""
WHERE ""Status"" = {0}";
SELECT ""Id"", ""BundleRequestId"", ""UserId"", ""Status"", ""JobType"", ""Priority"",
""ConfigJson"", ""StartDate"", ""EndDate"", ""ProgressPercentage"",
""AssignedWorkerId"", ""LastHeartbeat"", ""CreatedAt"", ""StartedAt"",
""CompletedAt"", ""ResultJson"", ""ErrorMessage"", ""RequestId"",
""GeneticRequestId"", ""RetryCount"", ""MaxRetries"", ""RetryAfter"",
""IsRetryable"", ""FailureCategory""
FROM ""Jobs""
WHERE ""Status"" = @status";
var parameters = new List<object> { (int)JobStatus.Pending };
var parameters = new List<NpgsqlParameter>
{
new NpgsqlParameter("status", NpgsqlDbType.Integer) { Value = (int)JobStatus.Pending }
};
if (jobType.HasValue)
{
sql += @" AND ""JobType"" = {1}";
parameters.Add((int)jobType.Value);
sql += @" AND ""JobType"" = @jobType";
parameters.Add(new NpgsqlParameter("jobType", NpgsqlDbType.Integer) { Value = (int)jobType.Value });
}
sql += @"
@@ -57,18 +70,61 @@ public class PostgreSqlJobRepository : IJobRepository
LIMIT 1
FOR UPDATE SKIP LOCKED";
// Use raw SQL with FromSqlRaw to get the next job with row-level locking
var job = await _context.Jobs
.FromSqlRaw(sql, parameters.ToArray())
.FirstOrDefaultAsync();
_logger.LogDebug("Claiming job with SQL: {Sql}, Parameters: Status={Status}, JobType={JobType}",
sql, (int)JobStatus.Pending, jobType.HasValue ? (int)jobType.Value : (int?)null);
// Execute raw SQL using ADO.NET to get the job with row-level locking
var connection = _context.Database.GetDbConnection();
await using var command = connection.CreateCommand();
command.Transaction = transaction.GetDbTransaction();
command.CommandText = sql;
command.Parameters.AddRange(parameters.ToArray());
JobEntity? job = null;
await using var reader = await command.ExecuteReaderAsync();
if (await reader.ReadAsync())
{
job = new JobEntity
{
Id = reader.GetGuid(reader.GetOrdinal("Id")),
BundleRequestId = reader.IsDBNull(reader.GetOrdinal("BundleRequestId")) ? null : reader.GetGuid(reader.GetOrdinal("BundleRequestId")),
UserId = reader.GetInt32(reader.GetOrdinal("UserId")),
Status = reader.GetInt32(reader.GetOrdinal("Status")),
JobType = reader.GetInt32(reader.GetOrdinal("JobType")),
Priority = reader.GetInt32(reader.GetOrdinal("Priority")),
ConfigJson = reader.GetString(reader.GetOrdinal("ConfigJson")),
StartDate = reader.GetDateTime(reader.GetOrdinal("StartDate")),
EndDate = reader.GetDateTime(reader.GetOrdinal("EndDate")),
ProgressPercentage = reader.GetInt32(reader.GetOrdinal("ProgressPercentage")),
AssignedWorkerId = reader.IsDBNull(reader.GetOrdinal("AssignedWorkerId")) ? null : reader.GetString(reader.GetOrdinal("AssignedWorkerId")),
LastHeartbeat = reader.IsDBNull(reader.GetOrdinal("LastHeartbeat")) ? null : reader.GetDateTime(reader.GetOrdinal("LastHeartbeat")),
CreatedAt = reader.GetDateTime(reader.GetOrdinal("CreatedAt")),
StartedAt = reader.IsDBNull(reader.GetOrdinal("StartedAt")) ? null : reader.GetDateTime(reader.GetOrdinal("StartedAt")),
CompletedAt = reader.IsDBNull(reader.GetOrdinal("CompletedAt")) ? null : reader.GetDateTime(reader.GetOrdinal("CompletedAt")),
ResultJson = reader.IsDBNull(reader.GetOrdinal("ResultJson")) ? null : reader.GetString(reader.GetOrdinal("ResultJson")),
ErrorMessage = reader.IsDBNull(reader.GetOrdinal("ErrorMessage")) ? null : reader.GetString(reader.GetOrdinal("ErrorMessage")),
RequestId = reader.IsDBNull(reader.GetOrdinal("RequestId")) ? null : reader.GetString(reader.GetOrdinal("RequestId")),
GeneticRequestId = reader.IsDBNull(reader.GetOrdinal("GeneticRequestId")) ? null : reader.GetString(reader.GetOrdinal("GeneticRequestId")),
RetryCount = reader.GetInt32(reader.GetOrdinal("RetryCount")),
MaxRetries = reader.GetInt32(reader.GetOrdinal("MaxRetries")),
RetryAfter = reader.IsDBNull(reader.GetOrdinal("RetryAfter")) ? null : reader.GetDateTime(reader.GetOrdinal("RetryAfter")),
IsRetryable = reader.GetBoolean(reader.GetOrdinal("IsRetryable")),
FailureCategory = reader.IsDBNull(reader.GetOrdinal("FailureCategory")) ? null : reader.GetInt32(reader.GetOrdinal("FailureCategory"))
};
}
await reader.CloseAsync();
if (job == null)
{
await transaction.RollbackAsync();
_logger.LogDebug("No job found to claim for worker {WorkerId}", workerId);
await transaction.CommitAsync();
return null;
}
// Update the job status atomically
// Attach and update the job entity
_context.Jobs.Attach(job);
job.Status = (int)JobStatus.Running;
job.AssignedWorkerId = workerId;
job.StartedAt = DateTime.UtcNow;
@@ -77,6 +133,7 @@ public class PostgreSqlJobRepository : IJobRepository
await _context.SaveChangesAsync();
await transaction.CommitAsync();
_logger.LogInformation("Claimed job {JobId} for worker {WorkerId}", job.Id, workerId);
return MapToDomain(job);
}
catch (Exception ex)
@@ -496,4 +553,3 @@ public class PostgreSqlJobRepository : IJobRepository
};
}
}

12
src/Managing.Workers.Api/Dockerfile
View File

@@ -7,7 +7,7 @@ FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
WORKDIR /src
# Copy project files for dependency restoration
COPY ["Managing.Workers.Api/Managing.Workers.Api.csproj", "Managing.Workers.Api/"]
COPY ["Managing.Workers/Managing.Workers.csproj", "Managing.Workers/"]
COPY ["Managing.Bootstrap/Managing.Bootstrap.csproj", "Managing.Bootstrap/"]
COPY ["Managing.Application/Managing.Application.csproj", "Managing.Application/"]
COPY ["Managing.Application.Abstractions/Managing.Application.Abstractions.csproj", "Managing.Application.Abstractions/"]
@@ -21,19 +21,19 @@ COPY ["Managing.Infrastructure.Storage/Managing.Infrastructure.Storage.csproj",
COPY ["Managing.Infrastructure.Web3/Managing.Infrastructure.Evm.csproj", "Managing.Infrastructure.Web3/"]
# Restore dependencies for all projects
RUN dotnet restore "Managing.Workers.Api/Managing.Workers.Api.csproj"
RUN dotnet restore "Managing.Workers/Managing.Workers.csproj"
# Copy everything else and build
COPY . .
WORKDIR "/src/Managing.Workers.Api"
RUN dotnet build "Managing.Workers.Api.csproj" -c Release -o /app/build
WORKDIR "/src/Managing.Workers"
RUN dotnet build "Managing.Workers.csproj" -c Release -o /app/build
FROM build AS publish
RUN dotnet publish "Managing.Workers.Api.csproj" -c Release -o /app/publish
RUN dotnet publish "Managing.Workers.csproj" -c Release -o /app/publish
FROM base AS final
WORKDIR /app
COPY --from=publish /app/publish .
ENTRYPOINT ["dotnet", "Managing.Workers.Api.dll"]
ENTRYPOINT ["dotnet", "Managing.Workers.dll"]

3
src/Managing.Workers.Api/Managing.Workers.Api.csproj → src/Managing.Workers.Api/Managing.Workers.csproj
View File

@@ -4,7 +4,7 @@
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<UserSecretsId>dotnet-Managing.Workers.Api-ff3f3987-4da4-4140-9180-b84c9e07b25f</UserSecretsId>
<UserSecretsId>dotnet-Managing.Workers-ff3f3987-4da4-4140-9180-b84c9e07b25f</UserSecretsId>
</PropertyGroup>
<ItemGroup>
@@ -24,3 +24,4 @@
<ProjectReference Include="..\Managing.Common\Managing.Common.csproj"/>
</ItemGroup>
</Project>

171
src/Managing.Workers.Api/Program.cs
View File

@@ -1,171 +0,0 @@
using Managing.Application.Workers;
using Managing.Bootstrap;
using Managing.Common;
using Managing.Infrastructure.Databases.InfluxDb.Models;
using Managing.Infrastructure.Databases.PostgreSql;
using Managing.Infrastructure.Databases.PostgreSql.Configurations;
using Microsoft.EntityFrameworkCore;
var host = Host.CreateDefaultBuilder(args)
.ConfigureAppConfiguration((hostingContext, config) =>
{
config.SetBasePath(AppContext.BaseDirectory);
config.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
.AddJsonFile($"appsettings.{hostingContext.HostingEnvironment.EnvironmentName}.json", optional: true)
.AddJsonFile("appsettings.SandboxLocal.json", optional: true, reloadOnChange: true)
.AddJsonFile("appsettings.ProductionLocal.json", optional: true, reloadOnChange: true)
.AddEnvironmentVariables()
.AddUserSecrets<Program>();
})
.ConfigureServices((hostContext, services) =>
{
var configuration = hostContext.Configuration;
// Initialize Sentry
SentrySdk.Init(options =>
{
options.Dsn = configuration["Sentry:Dsn"];
options.Debug = false;
options.SendDefaultPii = true;
options.AutoSessionTracking = true;
options.IsGlobalModeEnabled = false;
options.TracesSampleRate = 0.1;
options.Environment = hostContext.HostingEnvironment.EnvironmentName;
});
// Configure database
var postgreSqlConnectionString = configuration.GetSection(Constants.Databases.PostgreSql)["ConnectionString"];
services.Configure<PostgreSqlSettings>(configuration.GetSection(Constants.Databases.PostgreSql));
services.Configure<InfluxDbSettings>(configuration.GetSection(Constants.Databases.InfluxDb));
// Add DbContext
services.AddDbContext<ManagingDbContext>((serviceProvider, options) =>
{
options.UseNpgsql(postgreSqlConnectionString, npgsqlOptions =>
{
npgsqlOptions.CommandTimeout(60);
npgsqlOptions.EnableRetryOnFailure(maxRetryCount: 5, maxRetryDelay: TimeSpan.FromSeconds(10), errorCodesToAdd: null);
});
if (hostContext.HostingEnvironment.IsDevelopment())
{
options.EnableDetailedErrors();
options.EnableSensitiveDataLogging();
options.LogTo(Console.WriteLine, LogLevel.Information); // Enable SQL logging to debug table name issues
}
options.UseQueryTrackingBehavior(QueryTrackingBehavior.NoTracking);
options.EnableServiceProviderCaching();
}, ServiceLifetime.Scoped);
// Register compute dependencies (no Orleans)
services.RegisterComputeDependencies(configuration);
// Configure BacktestComputeWorker options
services.Configure<BacktestComputeWorkerOptions>(
configuration.GetSection(BacktestComputeWorkerOptions.SectionName));
// Get task slot from CapRover ({{.Task.Slot}}) or environment variable
// This identifies which instance of the worker is running
var taskSlot = Environment.GetEnvironmentVariable("TASK_SLOT") ??
Environment.GetEnvironmentVariable("CAPROVER_TASK_SLOT") ??
"0";
// Override WorkerId from environment variable if provided, otherwise use task slot
var workerId = Environment.GetEnvironmentVariable("WORKER_ID") ??
configuration["BacktestComputeWorker:WorkerId"] ??
$"{Environment.MachineName}-{taskSlot}";
services.Configure<BacktestComputeWorkerOptions>(options =>
{
options.WorkerId = workerId;
});
// Configure GeneticComputeWorker options
services.Configure<GeneticComputeWorkerOptions>(
configuration.GetSection(GeneticComputeWorkerOptions.SectionName));
// Override Genetic WorkerId from environment variable if provided, otherwise use task slot
var geneticWorkerId = Environment.GetEnvironmentVariable("GENETIC_WORKER_ID") ??
configuration["GeneticComputeWorker:WorkerId"] ??
$"{Environment.MachineName}-genetic-{taskSlot}";
services.Configure<GeneticComputeWorkerOptions>(options =>
{
options.WorkerId = geneticWorkerId;
});
// Register the backtest compute worker if enabled
var isBacktestWorkerEnabled = configuration.GetValue<bool>("WorkerBacktestCompute", false);
if (isBacktestWorkerEnabled)
{
services.AddHostedService<BacktestComputeWorker>();
}
// Register the genetic compute worker if enabled
var isGeneticWorkerEnabled = configuration.GetValue<bool>("WorkerGeneticCompute", false);
if (isGeneticWorkerEnabled)
{
services.AddHostedService<GeneticComputeWorker>();
}
})
.ConfigureLogging((hostingContext, logging) =>
{
logging.ClearProviders();
logging.AddConsole();
logging.AddConfiguration(hostingContext.Configuration.GetSection("Logging"));
})
.Build();
// Log worker status
var logger = host.Services.GetRequiredService<ILogger<Program>>();
var config = host.Services.GetRequiredService<IConfiguration>();
var isBacktestWorkerEnabled = config.GetValue<bool>("WorkerBacktestCompute", false);
var isGeneticWorkerEnabled = config.GetValue<bool>("WorkerGeneticCompute", false);
if (isBacktestWorkerEnabled)
{
var taskSlot = Environment.GetEnvironmentVariable("TASK_SLOT") ??
Environment.GetEnvironmentVariable("CAPROVER_TASK_SLOT") ??
"0";
var backtestWorkerId = Environment.GetEnvironmentVariable("WORKER_ID") ??
config["BacktestComputeWorker:WorkerId"] ??
$"{Environment.MachineName}-{taskSlot}";
logger.LogInformation("BacktestComputeWorker is enabled and will be started.");
logger.LogInformation("Backtest Worker ID: {WorkerId} (Task Slot: {TaskSlot})", backtestWorkerId, taskSlot);
}
else
{
logger.LogWarning("BacktestComputeWorker is disabled via configuration. No backtest jobs will be processed.");
}
if (isGeneticWorkerEnabled)
{
var taskSlot = Environment.GetEnvironmentVariable("TASK_SLOT") ??
Environment.GetEnvironmentVariable("CAPROVER_TASK_SLOT") ??
"0";
var geneticWorkerId = Environment.GetEnvironmentVariable("GENETIC_WORKER_ID") ??
config["GeneticComputeWorker:WorkerId"] ??
$"{Environment.MachineName}-genetic-{taskSlot}";
logger.LogInformation("GeneticComputeWorker is enabled and will be started.");
logger.LogInformation("Genetic Worker ID: {WorkerId} (Task Slot: {TaskSlot})", geneticWorkerId, taskSlot);
}
else
{
logger.LogWarning("GeneticComputeWorker is disabled via configuration. No genetic jobs will be processed.");
}
try
{
await host.RunAsync();
}
catch (Exception ex)
{
logger.LogCritical(ex, "Application terminated unexpectedly");
SentrySdk.CaptureException(ex);
throw;
}
finally
{
SentrySdk.FlushAsync(TimeSpan.FromSeconds(2)).Wait();
}

2
src/Managing.Workers.Api/Worker.cs
View File

@@ -1,4 +1,4 @@
namespace Managing.Workers.Api;
namespace Managing.Workers;
public class Worker : BackgroundService
{

39
src/Managing.Workers/Dockerfile Normal file
View File

@@ -0,0 +1,39 @@
# Use the official Microsoft .NET runtime as the base image (no ASP.NET needed for console worker)
FROM mcr.microsoft.com/dotnet/runtime:8.0 AS base
WORKDIR /app
# Use the official Microsoft .NET SDK image to build the code.
FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
WORKDIR /src
# Copy project files for dependency restoration
COPY ["Managing.Workers/Managing.Workers.csproj", "Managing.Workers/"]
COPY ["Managing.Bootstrap/Managing.Bootstrap.csproj", "Managing.Bootstrap/"]
COPY ["Managing.Application/Managing.Application.csproj", "Managing.Application/"]
COPY ["Managing.Application.Abstractions/Managing.Application.Abstractions.csproj", "Managing.Application.Abstractions/"]
COPY ["Managing.Common/Managing.Common.csproj", "Managing.Common/"]
COPY ["Managing.Core/Managing.Core.csproj", "Managing.Core/"]
COPY ["Managing.Domain/Managing.Domain.csproj", "Managing.Domain/"]
COPY ["Managing.Infrastructure.Database/Managing.Infrastructure.Databases.csproj", "Managing.Infrastructure.Database/"]
COPY ["Managing.Infrastructure.Exchanges/Managing.Infrastructure.Exchanges.csproj", "Managing.Infrastructure.Exchanges/"]
COPY ["Managing.Infrastructure.Messengers/Managing.Infrastructure.Messengers.csproj", "Managing.Infrastructure.Messengers/"]
COPY ["Managing.Infrastructure.Storage/Managing.Infrastructure.Storage.csproj", "Managing.Infrastructure.Storage/"]
COPY ["Managing.Infrastructure.Web3/Managing.Infrastructure.Evm.csproj", "Managing.Infrastructure.Web3/"]
# Restore dependencies for all projects
RUN dotnet restore "Managing.Workers/Managing.Workers.csproj"
# Copy everything else and build
COPY . .
WORKDIR "/src/Managing.Workers"
RUN dotnet build "Managing.Workers.csproj" -c Release -o /app/build
FROM build AS publish
RUN dotnet publish "Managing.Workers.csproj" -c Release -o /app/publish
FROM base AS final
WORKDIR /app
COPY --from=publish /app/publish .
ENTRYPOINT ["dotnet", "Managing.Workers.dll"]

27
src/Managing.Workers/Managing.Workers.csproj Normal file
View File

@@ -0,0 +1,27 @@
<Project Sdk="Microsoft.NET.Sdk.Worker">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<UserSecretsId>dotnet-Managing.Workers-ff3f3987-4da4-4140-9180-b84c9e07b25f</UserSecretsId>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.1"/>
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.11"/>
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.10"/>
<PackageReference Include="Sentry" Version="5.5.1"/>
<PackageReference Include="Microsoft.AspNetCore.SignalR.Core" Version="1.1.0"/>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Managing.Bootstrap\Managing.Bootstrap.csproj"/>
<ProjectReference Include="..\Managing.Application\Managing.Application.csproj"/>
<ProjectReference Include="..\Managing.Application.Abstractions\Managing.Application.Abstractions.csproj"/>
<ProjectReference Include="..\Managing.Infrastructure.Database\Managing.Infrastructure.Databases.csproj"/>
<ProjectReference Include="..\Managing.Infrastructure.Exchanges\Managing.Infrastructure.Exchanges.csproj"/>
<ProjectReference Include="..\Managing.Common\Managing.Common.csproj"/>
</ItemGroup>
</Project>

295
src/Managing.Workers/Program.cs Normal file
View File

@@ -0,0 +1,295 @@
using Managing.Application.Workers;
using Managing.Bootstrap;
using Managing.Common;
using Managing.Infrastructure.Databases.InfluxDb.Models;
using Managing.Infrastructure.Databases.PostgreSql;
using Managing.Infrastructure.Databases.PostgreSql.Configurations;
using Microsoft.EntityFrameworkCore;
// Explicitly set the environment before creating the host builder
var environment = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT")
?? Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT")
?? "Development";
// Log environment detection before host creation
var dotnetEnv = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
var aspnetcoreEnv = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT");
Console.WriteLine("═══════════════════════════════════════════════════════════");
Console.WriteLine("🔧 Environment Configuration");
Console.WriteLine("═══════════════════════════════════════════════════════════");
Console.WriteLine($" DOTNET_ENVIRONMENT: {dotnetEnv ?? "(not set)"}");
Console.WriteLine($" ASPNETCORE_ENVIRONMENT: {aspnetcoreEnv ?? "(not set)"}");
Console.WriteLine($" Selected Environment: {environment}");
Console.WriteLine("═══════════════════════════════════════════════════════════");
var host = Host.CreateDefaultBuilder(args)
.UseEnvironment(environment) // Explicitly set the environment
.ConfigureAppConfiguration((hostingContext, config) =>
{
var detectedEnv = hostingContext.HostingEnvironment.EnvironmentName;
Console.WriteLine($" Detected Environment: {detectedEnv}");
if (detectedEnv != environment)
{
Console.WriteLine($" ⚠️ WARNING: Environment mismatch! Expected: {environment}, Got: {detectedEnv}");
}
config.SetBasePath(AppContext.BaseDirectory);
// Load configuration files in order (later files override earlier ones)
// 1. Base appsettings.json (always loaded)
config.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true);
Console.WriteLine(" ✓ Loaded: appsettings.json");
// 2. Load ONLY the environment-specific file (not other environments)
if (!string.IsNullOrEmpty(detectedEnv))
{
var envFile = $"appsettings.{detectedEnv}.json";
config.AddJsonFile(envFile, optional: true, reloadOnChange: true);
Console.WriteLine($" ✓ Loaded: {envFile} (optional)");
}
// 3. Environment variables and user secrets (highest priority)
config.AddEnvironmentVariables()
.AddUserSecrets<Program>();
Console.WriteLine("═══════════════════════════════════════════════════════════");
})
.ConfigureServices((hostContext, services) =>
{
var configuration = hostContext.Configuration;
// Initialize Sentry
SentrySdk.Init(options =>
{
options.Dsn = configuration["Sentry:Dsn"];
options.Debug = false;
options.SendDefaultPii = true;
options.AutoSessionTracking = true;
options.IsGlobalModeEnabled = false;
options.TracesSampleRate = 0.1;
options.Environment = hostContext.HostingEnvironment.EnvironmentName;
});
// Configure database
var postgreSqlConnectionString = configuration.GetSection(Constants.Databases.PostgreSql)["ConnectionString"];
// Log database connection details (mask password for security)
if (!string.IsNullOrEmpty(postgreSqlConnectionString))
{
var connectionParts = postgreSqlConnectionString.Split(';')
.Where(p => !string.IsNullOrWhiteSpace(p))
.Select(p => p.Trim())
.ToDictionary(
p => p.Split('=')[0].Trim(),
p => p.Contains('=') ? p.Substring(p.IndexOf('=') + 1).Trim() : string.Empty,
StringComparer.OrdinalIgnoreCase);
var host = connectionParts.GetValueOrDefault("Host", "unknown");
var port = connectionParts.GetValueOrDefault("Port", "unknown");
var database = connectionParts.GetValueOrDefault("Database", "unknown");
var username = connectionParts.GetValueOrDefault("Username", "unknown");
var maskedConnectionString = postgreSqlConnectionString
.Replace(connectionParts.GetValueOrDefault("Password", ""), "***", StringComparison.OrdinalIgnoreCase);
Console.WriteLine("═══════════════════════════════════════════════════════════");
Console.WriteLine("📊 PostgreSQL Database Configuration");
Console.WriteLine("═══════════════════════════════════════════════════════════");
Console.WriteLine($" Host: {host}");
Console.WriteLine($" Port: {port}");
Console.WriteLine($" Database: {database}");
Console.WriteLine($" Username: {username}");
Console.WriteLine($" Connection String: {maskedConnectionString}");
Console.WriteLine("═══════════════════════════════════════════════════════════");
}
else
{
Console.WriteLine("⚠️ WARNING: PostgreSQL connection string is empty or not configured!");
}
services.Configure<PostgreSqlSettings>(configuration.GetSection(Constants.Databases.PostgreSql));
services.Configure<InfluxDbSettings>(configuration.GetSection(Constants.Databases.InfluxDb));
// Add DbContext
services.AddDbContext<ManagingDbContext>((serviceProvider, options) =>
{
options.UseNpgsql(postgreSqlConnectionString, npgsqlOptions =>
{
npgsqlOptions.CommandTimeout(60);
npgsqlOptions.EnableRetryOnFailure(maxRetryCount: 5, maxRetryDelay: TimeSpan.FromSeconds(10), errorCodesToAdd: null);
});
if (hostContext.HostingEnvironment.IsDevelopment())
{
options.EnableDetailedErrors();
options.EnableSensitiveDataLogging();
// SQL logging disabled - uncomment below line if needed for debugging
// options.LogTo(Console.WriteLine, LogLevel.Information);
}
options.UseQueryTrackingBehavior(QueryTrackingBehavior.NoTracking);
options.EnableServiceProviderCaching();
}, ServiceLifetime.Scoped);
// Register compute dependencies (no Orleans)
services.RegisterComputeDependencies(configuration);
// Configure BacktestComputeWorker options
services.Configure<BacktestComputeWorkerOptions>(
configuration.GetSection(BacktestComputeWorkerOptions.SectionName));
// Get task slot from CapRover ({{.Task.Slot}}) or environment variable
// This identifies which instance of the worker is running
var taskSlot = Environment.GetEnvironmentVariable("TASK_SLOT") ??
Environment.GetEnvironmentVariable("CAPROVER_TASK_SLOT") ??
"0";
// Override WorkerId from environment variable if provided, otherwise use task slot
var workerId = Environment.GetEnvironmentVariable("WORKER_ID") ??
configuration["BacktestComputeWorker:WorkerId"] ??
$"{Environment.MachineName}-{taskSlot}";
services.Configure<BacktestComputeWorkerOptions>(options =>
{
options.WorkerId = workerId;
});
// Configure GeneticComputeWorker options
services.Configure<GeneticComputeWorkerOptions>(
configuration.GetSection(GeneticComputeWorkerOptions.SectionName));
// Override Genetic WorkerId from environment variable if provided, otherwise use task slot
var geneticWorkerId = Environment.GetEnvironmentVariable("GENETIC_WORKER_ID") ??
configuration["GeneticComputeWorker:WorkerId"] ??
$"{Environment.MachineName}-genetic-{taskSlot}";
services.Configure<GeneticComputeWorkerOptions>(options =>
{
options.WorkerId = geneticWorkerId;
});
// Register the backtest compute worker if enabled
var isBacktestWorkerEnabled = configuration.GetValue<bool>("WorkerBacktestCompute", false);
if (isBacktestWorkerEnabled)
{
services.AddHostedService<BacktestComputeWorker>();
}
// Register the genetic compute worker if enabled
var isGeneticWorkerEnabled = configuration.GetValue<bool>("WorkerGeneticCompute", false);
if (isGeneticWorkerEnabled)
{
services.AddHostedService<GeneticComputeWorker>();
}
})
.ConfigureLogging((hostingContext, logging) =>
{
logging.ClearProviders();
logging.AddConsole();
logging.AddConfiguration(hostingContext.Configuration.GetSection("Logging"));
})
.Build();
// Log worker status
var logger = host.Services.GetRequiredService<ILogger<Program>>();
var config = host.Services.GetRequiredService<IConfiguration>();
// Log PostgreSQL connection details
var postgreSqlConnectionString = config.GetSection(Constants.Databases.PostgreSql)["ConnectionString"];
if (!string.IsNullOrEmpty(postgreSqlConnectionString))
{
var connectionParts = postgreSqlConnectionString.Split(';')
.Where(p => !string.IsNullOrWhiteSpace(p))
.Select(p => p.Trim())
.ToDictionary(
p => p.Split('=')[0].Trim(),
p => p.Contains('=') ? p.Substring(p.IndexOf('=') + 1).Trim() : string.Empty,
StringComparer.OrdinalIgnoreCase);
var hostName = connectionParts.GetValueOrDefault("Host", "unknown");
var port = connectionParts.GetValueOrDefault("Port", "unknown");
var database = connectionParts.GetValueOrDefault("Database", "unknown");
var username = connectionParts.GetValueOrDefault("Username", "unknown");
logger.LogInformation("═══════════════════════════════════════════════════════════");
logger.LogInformation("📊 PostgreSQL Database Configuration");
logger.LogInformation("═══════════════════════════════════════════════════════════");
logger.LogInformation(" Host: {Host}", hostName);
logger.LogInformation(" Port: {Port}", port);
logger.LogInformation(" Database: {Database}", database);
logger.LogInformation(" Username: {Username}", username);
logger.LogInformation("═══════════════════════════════════════════════════════════");
// Test database connection
try
{
using var scope = host.Services.CreateScope();
var dbContext = scope.ServiceProvider.GetRequiredService<ManagingDbContext>();
var canConnect = await dbContext.Database.CanConnectAsync();
if (canConnect)
{
logger.LogInformation("✅ Database connection test: SUCCESS");
}
else
{
logger.LogWarning("⚠️ Database connection test: FAILED - Cannot connect to database");
}
}
catch (Exception ex)
{
logger.LogError(ex, "❌ Database connection test: FAILED - {Error}", ex.Message);
}
}
else
{
logger.LogWarning("⚠️ WARNING: PostgreSQL connection string is empty or not configured!");
}
var isBacktestWorkerEnabled = config.GetValue<bool>("WorkerBacktestCompute", false);
var isGeneticWorkerEnabled = config.GetValue<bool>("WorkerGeneticCompute", false);
if (isBacktestWorkerEnabled)
{
var taskSlot = Environment.GetEnvironmentVariable("TASK_SLOT") ??
Environment.GetEnvironmentVariable("CAPROVER_TASK_SLOT") ??
"0";
var backtestWorkerId = Environment.GetEnvironmentVariable("WORKER_ID") ??
config["BacktestComputeWorker:WorkerId"] ??
$"{Environment.MachineName}-{taskSlot}";
logger.LogInformation("BacktestComputeWorker is enabled and will be started.");
logger.LogInformation("Backtest Worker ID: {WorkerId} (Task Slot: {TaskSlot})", backtestWorkerId, taskSlot);
}
else
{
logger.LogWarning("BacktestComputeWorker is disabled via configuration. No backtest jobs will be processed.");
}
if (isGeneticWorkerEnabled)
{
var taskSlot = Environment.GetEnvironmentVariable("TASK_SLOT") ??
Environment.GetEnvironmentVariable("CAPROVER_TASK_SLOT") ??
"0";
var geneticWorkerId = Environment.GetEnvironmentVariable("GENETIC_WORKER_ID") ??
config["GeneticComputeWorker:WorkerId"] ??
$"{Environment.MachineName}-genetic-{taskSlot}";
logger.LogInformation("GeneticComputeWorker is enabled and will be started.");
logger.LogInformation("Genetic Worker ID: {WorkerId} (Task Slot: {TaskSlot})", geneticWorkerId, taskSlot);
}
else
{
logger.LogWarning("GeneticComputeWorker is disabled via configuration. No genetic jobs will be processed.");
}
try
{
await host.RunAsync();
}
catch (Exception ex)
{
logger.LogCritical(ex, "Application terminated unexpectedly");
SentrySdk.CaptureException(ex);
throw;
}
finally
{
SentrySdk.FlushAsync(TimeSpan.FromSeconds(2)).Wait();
}

24
src/Managing.Workers/Worker.cs Normal file
View File

@@ -0,0 +1,24 @@
namespace Managing.Workers;
public class Worker : BackgroundService
{
private readonly ILogger<Worker> _logger;
public Worker(ILogger<Worker> logger)
{
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
if (_logger.IsEnabled(LogLevel.Information))
{
_logger.LogInformation("Worker running at: {time}", DateTimeOffset.Now);
}
await Task.Delay(1000, stoppingToken);
}
}
}

1
src/Managing.Workers.Api/appsettings.Development.json → src/Managing.Workers/appsettings.Development.json
View File

@@ -7,6 +7,7 @@
},
"WorkerBacktestCompute": true,
"BacktestComputeWorker": {
"WorkerId": "Oda-backtest-0",
"MaxConcurrentBacktests": 6,
"JobPollIntervalSeconds": 5,
"HeartbeatIntervalSeconds": 30,

0
src/Managing.Workers.Api/appsettings.ProductionLocal.json → src/Managing.Workers/appsettings.ProductionLocal.json
View File

0
src/Managing.Workers.Api/appsettings.SandboxLocal.json → src/Managing.Workers/appsettings.SandboxLocal.json
View File

8
src/Managing.Workers.Api/appsettings.json → src/Managing.Workers/appsettings.json
View File

@@ -19,14 +19,6 @@
"HeartbeatIntervalSeconds": 30,
"StaleJobTimeoutMinutes": 10
},
"PostgreSql": {
"ConnectionString": "Host=managing-postgre.apps.managing.live;Port=5432;Database=managing;Username=postgres;Password=29032b13a5bc4d37"
},
"InfluxDb": {
"Url": "https://influx-db.apps.managing.live",
"Organization": "managing-org",
"Token": "eOuXcXhH7CS13Iw4CTiDDpRjIjQtEVPOloD82pLPOejI4n0BsEj1YzUw0g3Cs1mdDG5m-RaxCavCMsVTtS5wIQ=="
},
"Sentry": {
"Dsn": "https://fe12add48c56419bbdfa86227c188e7a@glitch.kai.managing.live/1"
},

0
src/Managing.Workers.Api/captain-definition → src/Managing.Workers/captain-definition
View File

2
src/Managing.sln
View File

@@ -68,7 +68,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Managing.Aspire.ServiceDefa
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Managing.Nswag", "Managing.Nswag\Managing.Nswag.csproj", "{BE50F950-C1D4-4CE0-B32E-6AAC996770D5}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Managing.Workers.Api", "Managing.Workers.Api\Managing.Workers.Api.csproj", "{B7D66A73-CA3A-4DE5-8E88-59D50C4018A6}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Managing.Workers", "Managing.Workers\Managing.Workers.csproj", "{B7D66A73-CA3A-4DE5-8E88-59D50C4018A6}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution