Postgres (#30)

* Add postgres * Migrate users * Migrate geneticRequest * Try to fix Concurrent call * Fix asyncawait * Fix async and concurrent * Migrate backtests * Add cache for user by address * Fix backtest migration * Fix not open connection * Fix backtest command error * Fix concurrent * Fix all concurrency * Migrate TradingRepo * Fix scenarios * Migrate statistic repo * Save botbackup * Add settings et moneymanagement * Add bot postgres * fix a bit more backups * Fix bot model * Fix loading backup * Remove cache market for read positions * Add workers to postgre * Fix workers api * Reduce get Accounts for workers * Migrate synth to postgre * Fix backtest saved * Remove mongodb * botservice decorrelation * Fix tradingbot scope call * fix tradingbot * fix concurrent * Fix scope for genetics * Fix account over requesting * Fix bundle backtest worker * fix a lot of things * fix tab backtest * Remove optimized moneymanagement * Add light signal to not use User and too much property * Make money management lighter * insert indicators to awaitable * Migrate add strategies to await * Refactor scenario and indicator retrieval to use asynchronous methods throughout the application * add more async await * Add services * Fix and clean * Fix bot a bit * Fix bot and add message for cooldown * Remove fees * Add script to deploy db * Update dfeeploy script * fix script * Add idempotent script and backup * finish script migration * Fix did user and agent name on start bot
2025-07-27 15:42:17 +02:00
parent 361bfbf6e8
commit 422fecea7b
294 changed files with 23953 additions and 7272 deletions
--- a/src/Managing.Api.Workers/Controllers/WorkerController.cs
+++ b/src/Managing.Api.Workers/Controllers/WorkerController.cs
@@ -18,9 +18,10 @@ public class WorkerController : ControllerBase
    }

    [HttpGet]
-    public ActionResult<List<Worker>> GetWorkers()
+    public async Task<ActionResult<List<Worker>>> GetWorkers()
    {
-        return Ok(_workerService.GetWorkers());
+        var workers = await _workerService.GetWorkers();
+        return Ok(workers.ToList());
    }

    [HttpPatch]
--- a/src/Managing.Api.Workers/Program.cs
+++ b/src/Managing.Api.Workers/Program.cs
@@ -6,10 +6,10 @@ using Managing.Bootstrap;
 using Managing.Common;
 using Managing.Core.Middleawares;
 using Managing.Infrastructure.Databases.InfluxDb.Models;
-using Managing.Infrastructure.Databases.MongoDb;
-using Managing.Infrastructure.Databases.MongoDb.Configurations;
+using Managing.Infrastructure.Databases.PostgreSql;
 using Managing.Infrastructure.Evm.Models.Privy;
 using Microsoft.AspNetCore.Diagnostics.HealthChecks;
+using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Diagnostics.HealthChecks;
 using Microsoft.OpenApi.Models;
 using NSwag;
@@ -27,9 +27,9 @@ builder.Configuration.SetBasePath(AppContext.BaseDirectory);
 builder.Configuration.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
    .AddJsonFile($"appsettings.{builder.Environment.EnvironmentName}.json");

-var mongoConnectionString = builder.Configuration.GetSection(Constants.Databases.MongoDb)["ConnectionString"];
 var influxUrl = builder.Configuration.GetSection(Constants.Databases.InfluxDb)["Url"];
 var web3ProxyUrl = builder.Configuration.GetSection("Web3Proxy")["BaseUrl"];
+var postgreSqlConnectionString = builder.Configuration.GetSection("PostgreSql")["ConnectionString"];

 // Initialize Sentry
 SentrySdk.Init(options =>
@@ -61,7 +61,6 @@ builder.Services.AddServiceDiscovery();
 // Configure health checks
 builder.Services.AddHealthChecks()
    .AddCheck("self", () => HealthCheckResult.Healthy(), ["live"])
-    .AddMongoDb(mongoConnectionString, name: "mongodb", tags: ["database"])
    .AddUrlGroup(new Uri($"{influxUrl}/health"), name: "influxdb", tags: ["database"])
    .AddUrlGroup(new Uri($"{web3ProxyUrl}/health"), name: "web3proxy", tags: ["api"]);

@@ -91,7 +90,6 @@ builder.Host.UseSerilog((hostBuilder, loggerConfiguration) =>
        .WriteTo.Elasticsearch(es);
 });
 builder.Services.AddOptions();
-builder.Services.Configure<ManagingDatabaseSettings>(builder.Configuration.GetSection(Constants.Databases.MongoDb));
 builder.Services.Configure<InfluxDbSettings>(builder.Configuration.GetSection(Constants.Databases.InfluxDb));
 builder.Services.Configure<PrivySettings>(builder.Configuration.GetSection(Constants.ThirdParty.Privy));
 builder.Services.AddControllers().AddJsonOptions(options =>
@@ -108,6 +106,27 @@ builder.Services.AddCors(o => o.AddPolicy("CorsPolicy", builder =>
 }));
 builder.Services.AddSignalR().AddJsonProtocol();

+// Add PostgreSQL DbContext for worker services
+builder.Services.AddDbContext<ManagingDbContext>(options =>
+{
+    options.UseNpgsql(postgreSqlConnectionString, npgsqlOptions =>
+    {
+        npgsqlOptions.CommandTimeout(60);
+        npgsqlOptions.EnableRetryOnFailure(maxRetryCount: 5, maxRetryDelay: TimeSpan.FromSeconds(10),
+            errorCodesToAdd: null);
+    });
+    if (builder.Environment.IsDevelopment())
+    {
+        options.EnableDetailedErrors();
+        options.EnableSensitiveDataLogging();
+        options.EnableThreadSafetyChecks();
+    }
+
+    options.UseQueryTrackingBehavior(QueryTrackingBehavior.NoTracking);
+    options.EnableServiceProviderCaching();
+    options.LogTo(msg => Console.WriteLine(msg), LogLevel.Warning);
+}, ServiceLifetime.Scoped);
+
 builder.Services.RegisterWorkersDependencies(builder.Configuration);
 builder.Services.AddEndpointsApiExplorer();
 builder.Services.AddOpenApiDocument(document =>
@@ -157,24 +176,6 @@ builder.WebHost.SetupDiscordBot();
 var app = builder.Build();
 app.UseSerilogRequestLogging();

-// Create MongoDB indexes on startup
-try
-{
-    var indexService = app.Services.GetRequiredService<IndexService>();
-    await indexService.CreateIndexesAsync();
-}
-catch (Exception ex)
-{
-    // Log the error but don't fail the application startup
-    var logger = app.Services.GetRequiredService<ILogger<Program>>();
-    logger.LogError(ex, "Failed to create MongoDB indexes on startup. The application will continue without indexes.");
-}
-
-if (app.Environment.IsDevelopment())
-{
-    app.UseDeveloperExceptionPage();
-}
-
 app.UseOpenApi();
 app.UseSwaggerUI(c =>
 {
--- a/src/Managing.Api.Workers/appsettings.Development.json
+++ b/src/Managing.Api.Workers/appsettings.Development.json
@@ -1,8 +1,4 @@
 {
-  "ManagingDatabase": {
-    "ConnectionString": "mongodb://localhost:27017",
-    "DatabaseName": "ManagingDb"
-  },
  "InfluxDb": {
    "Url": "http://localhost:8086/",
    "Token": ""
--- a/src/Managing.Api.Workers/appsettings.KaiServer.json
+++ b/src/Managing.Api.Workers/appsettings.KaiServer.json
@@ -1,8 +1,4 @@
 {
-  "ManagingDatabase": {
-    "ConnectionString": "mongodb://admin:r8oJiDIKbsEi@mongo-db.apps.managing.live:27017/?authMechanism=SCRAM-SHA-256",
-    "DatabaseName": "ManagingDb"
-  },
  "InfluxDb": {
    "Url": "https://influx-db.apps.managing.live",
    "Organization": "managing-org",
--- a/src/Managing.Api.Workers/appsettings.Oda-docker.json
+++ b/src/Managing.Api.Workers/appsettings.Oda-docker.json
@@ -1,8 +1,4 @@
 {
-  "ManagingDatabase": {
-    "ConnectionString": "mongodb://managingdb:27017",
-    "DatabaseName": "ManagingDb"
-  },
  "InfluxDb": {
    "Url": "http://influxdb:8086/",
    "Organization": "managing-org",
--- a/src/Managing.Api.Workers/appsettings.Oda.json
+++ b/src/Managing.Api.Workers/appsettings.Oda.json
@@ -1,8 +1,4 @@
 {
-  "ManagingDatabase": {
-    "ConnectionString": "mongodb://localhost:27017",
-    "DatabaseName": "ManagingDb"
-  },
  "InfluxDb": {
    "Url": "http://localhost:8086/",
    "Organization": "managing-org",
@@ -37,5 +33,6 @@
  "WorkerTraderWatcher": false,
  "WorkerLeaderboard": false,
  "WorkerFundingRatesWatcher": false,
-  "WorkerGeneticAlgorithm": true
+  "WorkerGeneticAlgorithm": false,
+  "WorkerBundleBacktest": true
 }
--- a/src/Managing.Api.Workers/appsettings.Production.json
+++ b/src/Managing.Api.Workers/appsettings.Production.json
@@ -1,8 +1,4 @@
 {
-  "ManagingDatabase": {
-    "ConnectionString": "mongodb://root:gRerdhtg546FgrW@srv-captain--mongo:27017/?authMechanism=SCRAM-SHA-256",
-    "DatabaseName": "ManagingDb"
-  },
  "InfluxDb": {
    "Url": "https://influx-db.apps.managing.live",
    "Organization": "managing-org",
--- a/src/Managing.Api.Workers/appsettings.Sandbox.json
+++ b/src/Managing.Api.Workers/appsettings.Sandbox.json
@@ -1,8 +1,4 @@
 {
-  "ManagingDatabase": {
-    "ConnectionString": "mongodb://admin:r8oJiDIKbsEi@srv-captain--mongo-db:27017/?authMechanism=SCRAM-SHA-256",
-    "DatabaseName": "ManagingDb"
-  },
  "InfluxDb": {
    "Url": "http://srv-captain--influx-db:8086/",
    "Organization": "managing-org",
--- a/src/Managing.Api.Workers/appsettings.SandboxLocal.json
+++ b/src/Managing.Api.Workers/appsettings.SandboxLocal.json
@@ -1,8 +1,4 @@
 {
-  "ManagingDatabase": {
-    "ConnectionString": "mongodb://admin:r8oJiDIKbsEi@mongo-db.apps.managing.live:27017/?authMechanism=SCRAM-SHA-256",
-    "DatabaseName": "ManagingDb"
-  },
  "InfluxDb": {
    "Url": "https://influx-db.apps.managing.live",
    "Organization": "managing-org",
--- a/src/Managing.Api.Workers/appsettings.json
+++ b/src/Managing.Api.Workers/appsettings.json
@@ -1,13 +1,12 @@
 {
-  "ManagingDatabase": {
-    "ConnectionString": "mongodb://managingdb",
-    "DatabaseName": "ManagingDb"
-  },
  "InfluxDb": {
    "Url": "http://influxdb:8086/",
    "Organization": "",
    "Token": ""
  },
+  "PostgreSql": {
+    "ConnectionString": "Host=localhost;Port=5432;Database=managing;Username=postgres;Password=postgres"
+  },
  "Serilog": {
    "MinimumLevel": {
      "Default": "Information",
@@ -17,6 +16,9 @@
      }
    }
  },
+  "N8n": {
+    "WebhookUrl": "https://n8n.kai.managing.live/webhook/fa9308b6-983b-42ec-b085-71599d655951"
+  },
  "ElasticConfiguration": {
    "Uri": "http://elasticsearch:9200"
  },