From eeb29236469a4d8d5e2311ad4524e421d4f076d0 Mon Sep 17 00:00:00 2001 From: cryptooda Date: Sat, 16 Aug 2025 05:09:04 +0700 Subject: [PATCH] Update silo/cluster config --- .../Controllers/WorkerController.cs | 32 --- src/Managing.Api.Workers/Dockerfile | 33 --- .../Filters/EnumSchemaFilter.cs | 20 -- .../Managing.Api.Workers.csproj | 60 ----- .../Middleware/SentryDiagnosticsMiddleware.cs | 90 ------- src/Managing.Api.Workers/Program.cs | 222 ------------------ .../appsettings.Development.json | 19 -- .../appsettings.KaiServer.json | 27 --- .../appsettings.Oda-docker.json | 24 -- src/Managing.Api.Workers/appsettings.Oda.json | 42 ---- .../appsettings.Production.json | 34 --- .../appsettings.Sandbox.json | 27 --- .../appsettings.SandboxLocal.json | 24 -- src/Managing.Api.Workers/appsettings.json | 53 ----- .../README-ORLEANS-TROUBLESHOOTING.md | 118 ++++++++++ src/Managing.Api/appsettings.Production.json | 5 + src/Managing.Bootstrap/ApiBootstrap.cs | 104 +++++--- src/Managing.Docker/docker-compose.yml | 11 +- 18 files changed, 205 insertions(+), 740 deletions(-) delete mode 100644 src/Managing.Api.Workers/Controllers/WorkerController.cs delete mode 100644 src/Managing.Api.Workers/Dockerfile delete mode 100644 src/Managing.Api.Workers/Filters/EnumSchemaFilter.cs delete mode 100644 src/Managing.Api.Workers/Managing.Api.Workers.csproj delete mode 100644 src/Managing.Api.Workers/Middleware/SentryDiagnosticsMiddleware.cs delete mode 100644 src/Managing.Api.Workers/Program.cs delete mode 100644 src/Managing.Api.Workers/appsettings.Development.json delete mode 100644 src/Managing.Api.Workers/appsettings.KaiServer.json delete mode 100644 src/Managing.Api.Workers/appsettings.Oda-docker.json delete mode 100644 src/Managing.Api.Workers/appsettings.Oda.json delete mode 100644 src/Managing.Api.Workers/appsettings.Production.json delete mode 100644 src/Managing.Api.Workers/appsettings.Sandbox.json delete mode 100644 src/Managing.Api.Workers/appsettings.SandboxLocal.json delete mode 100644 src/Managing.Api.Workers/appsettings.json create mode 100644 src/Managing.Api/README-ORLEANS-TROUBLESHOOTING.md diff --git a/src/Managing.Api.Workers/Controllers/WorkerController.cs b/src/Managing.Api.Workers/Controllers/WorkerController.cs deleted file mode 100644 index 1ca4948..0000000 --- a/src/Managing.Api.Workers/Controllers/WorkerController.cs +++ /dev/null @@ -1,32 +0,0 @@ -using Managing.Application.Abstractions.Services; -using Managing.Domain.Workers; -using Microsoft.AspNetCore.Mvc; -using static Managing.Common.Enums; - -namespace Managing.Api.Workers.Controllers; - -[ApiController] -[Route("[controller]")] -[Produces("application/json")] -public class WorkerController : ControllerBase -{ - private readonly IWorkerService _workerService; - - public WorkerController(IWorkerService workerService) - { - _workerService = workerService; - } - - [HttpGet] - public async Task>> GetWorkers() - { - var workers = await _workerService.GetWorkers(); - return Ok(workers.ToList()); - } - - [HttpPatch] - public async Task ToggleWorker(WorkerType workerType) - { - return Ok(await _workerService.ToggleWorker(workerType)); - } -} \ No newline at end of file diff --git a/src/Managing.Api.Workers/Dockerfile b/src/Managing.Api.Workers/Dockerfile deleted file mode 100644 index 0bb0e70..0000000 --- a/src/Managing.Api.Workers/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -# Use the official Microsoft ASP.NET Core runtime as the base image. -FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS base -WORKDIR /app -EXPOSE 80 -EXPOSE 443 - -# Use the official Microsoft .NET SDK image to build the code. -FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build -WORKDIR /src -COPY ["Managing.Api.Workers/Managing.Api.Workers.csproj", "Managing.Api.Workers/"] -COPY ["Managing.Bootstrap/Managing.Bootstrap.csproj", "Managing.Bootstrap/"] -COPY ["Managing.Infrastructure.Storage/Managing.Infrastructure.Storage.csproj", "Managing.Infrastructure.Storage/"] -COPY ["Managing.Application/Managing.Application.csproj", "Managing.Application/"] -COPY ["Managing.Common/Managing.Common.csproj", "Managing.Common/"] -COPY ["Managing.Core/Managing.Core.csproj", "Managing.Core/"] -COPY ["Managing.Application.Abstractions/Managing.Application.Abstractions.csproj", "Managing.Application.Abstractions/"] -COPY ["Managing.Domain/Managing.Domain.csproj", "Managing.Domain/"] -COPY ["Managing.Infrastructure.Messengers/Managing.Infrastructure.Messengers.csproj", "Managing.Infrastructure.Messengers/"] -COPY ["Managing.Infrastructure.Exchanges/Managing.Infrastructure.Exchanges.csproj", "Managing.Infrastructure.Exchanges/"] -COPY ["Managing.Infrastructure.Database/Managing.Infrastructure.Databases.csproj", "Managing.Infrastructure.Database/"] -RUN dotnet restore "Managing.Api.Workers/Managing.Api.Workers.csproj" -COPY . . -WORKDIR "/src/Managing.Api.Workers" -RUN dotnet build "Managing.Api.Workers.csproj" -c Release -o /app/build - -FROM build AS publish -RUN dotnet publish "Managing.Api.Workers.csproj" -c Release -o /app/publish - -FROM base AS final -WORKDIR /app -COPY --from=publish /app/publish . -COPY managing_cert.pfx . -ENTRYPOINT ["dotnet", "Managing.Api.Workers.dll"] diff --git a/src/Managing.Api.Workers/Filters/EnumSchemaFilter.cs b/src/Managing.Api.Workers/Filters/EnumSchemaFilter.cs deleted file mode 100644 index 063f4ae..0000000 --- a/src/Managing.Api.Workers/Filters/EnumSchemaFilter.cs +++ /dev/null @@ -1,20 +0,0 @@ -using Microsoft.OpenApi.Any; -using Microsoft.OpenApi.Models; -using Swashbuckle.AspNetCore.SwaggerGen; - -namespace Managing.Api.Workers.Filters -{ - public class EnumSchemaFilter : ISchemaFilter - { - public void Apply(OpenApiSchema model, SchemaFilterContext context) - { - if (context.Type.IsEnum) - { - model.Enum.Clear(); - Enum.GetNames(context.Type) - .ToList() - .ForEach(n => model.Enum.Add(new OpenApiString(n))); - } - } - } -} diff --git a/src/Managing.Api.Workers/Managing.Api.Workers.csproj b/src/Managing.Api.Workers/Managing.Api.Workers.csproj deleted file mode 100644 index 2482284..0000000 --- a/src/Managing.Api.Workers/Managing.Api.Workers.csproj +++ /dev/null @@ -1,60 +0,0 @@ - - - - net8.0 - enable - AnyCPU;x64 - 3900ce93-de15-49e5-9a61-7dc2209939ca - Linux - ..\.. - ..\..\docker-compose.dcproj - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Always - - - Always - - - Always - - - Always - - - Always - - - Always - - - diff --git a/src/Managing.Api.Workers/Middleware/SentryDiagnosticsMiddleware.cs b/src/Managing.Api.Workers/Middleware/SentryDiagnosticsMiddleware.cs deleted file mode 100644 index 2d5a53a..0000000 --- a/src/Managing.Api.Workers/Middleware/SentryDiagnosticsMiddleware.cs +++ /dev/null @@ -1,90 +0,0 @@ -using Sentry; -using System.Text; - -namespace Managing.Api.Workers.Middleware -{ - public class SentryDiagnosticsMiddleware - { - private readonly RequestDelegate _next; - private readonly ILogger _logger; - - public SentryDiagnosticsMiddleware(RequestDelegate next, ILogger logger) - { - _next = next; - _logger = logger; - } - - public async Task InvokeAsync(HttpContext context) - { - // Only activate for the /api/sentry-diagnostics endpoint - if (context.Request.Path.StartsWithSegments("/api/sentry-diagnostics")) - { - await HandleDiagnosticsRequest(context); - return; - } - - await _next(context); - } - - private async Task HandleDiagnosticsRequest(HttpContext context) - { - var response = new StringBuilder(); - response.AppendLine("Sentry Diagnostics Report"); - response.AppendLine("========================"); - response.AppendLine($"Timestamp: {DateTime.Now}"); - response.AppendLine(); - - // Check if Sentry is initialized - response.AppendLine("## Sentry SDK Status"); - response.AppendLine($"Sentry Enabled: {SentrySdk.IsEnabled}"); - response.AppendLine($"Application Environment: {Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT")}"); - response.AppendLine(); - - // Send a test event - response.AppendLine("## Test Event"); - try - { - var id = SentrySdk.CaptureMessage($"Diagnostics test from {context.Request.Host} at {DateTime.Now}", SentryLevel.Info); - response.AppendLine($"Test Event ID: {id}"); - response.AppendLine("Test event was sent to Sentry. Check your Sentry dashboard to confirm it was received."); - - // Try to send an exception too - try - { - throw new Exception("Test exception from diagnostics middleware"); - } - catch (Exception ex) - { - var exceptionId = SentrySdk.CaptureException(ex); - response.AppendLine($"Test Exception ID: {exceptionId}"); - } - } - catch (Exception ex) - { - response.AppendLine($"Error sending test event: {ex.Message}"); - response.AppendLine(ex.StackTrace); - } - - response.AppendLine(); - response.AppendLine("## Connectivity Check"); - response.AppendLine("If events are not appearing in Sentry, check the following:"); - response.AppendLine("1. Verify your DSN is correct in appsettings.json"); - response.AppendLine("2. Ensure your network allows outbound HTTPS connections to sentry.apps.managing.live"); - response.AppendLine("3. Check Sentry server logs for any ingestion issues"); - response.AppendLine("4. Verify your Sentry project is correctly configured to receive events"); - - // Return the diagnostic information - context.Response.ContentType = "text/plain"; - await context.Response.WriteAsync(response.ToString()); - } - } - - // Extension method used to add the middleware to the HTTP request pipeline. - public static class SentryDiagnosticsMiddlewareExtensions - { - public static IApplicationBuilder UseSentryDiagnostics(this IApplicationBuilder builder) - { - return builder.UseMiddleware(); - } - } -} \ No newline at end of file diff --git a/src/Managing.Api.Workers/Program.cs b/src/Managing.Api.Workers/Program.cs deleted file mode 100644 index e44ef65..0000000 --- a/src/Managing.Api.Workers/Program.cs +++ /dev/null @@ -1,222 +0,0 @@ -// DEPRECATED: This Workers API project has been consolidated into the main Managing.Api project -// All workers are now hosted as background services in the main API -// This project is kept for reference but should not be deployed - -using System.Text.Json.Serialization; -using HealthChecks.UI.Client; -using Managing.Api.Workers.Filters; -using Managing.Application.Hubs; -using Managing.Bootstrap; -using Managing.Common; -using Managing.Core.Middleawares; -using Managing.Infrastructure.Databases.InfluxDb.Models; -using Managing.Infrastructure.Databases.PostgreSql; -using Managing.Infrastructure.Evm.Models.Privy; -using Microsoft.AspNetCore.Diagnostics.HealthChecks; -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Diagnostics.HealthChecks; -using Microsoft.OpenApi.Models; -using NSwag; -using NSwag.Generation.Processors.Security; -using Serilog; -using Serilog.Events; -using Serilog.Sinks.Elasticsearch; -using OpenApiSecurityRequirement = Microsoft.OpenApi.Models.OpenApiSecurityRequirement; -using OpenApiSecurityScheme = NSwag.OpenApiSecurityScheme; - -// Builder -var builder = WebApplication.CreateBuilder(args); - -builder.Configuration.SetBasePath(AppContext.BaseDirectory); -builder.Configuration.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) - .AddJsonFile($"appsettings.{builder.Environment.EnvironmentName}.json"); - -var influxUrl = builder.Configuration.GetSection(Constants.Databases.InfluxDb)["Url"]; -var web3ProxyUrl = builder.Configuration.GetSection("Web3Proxy")["BaseUrl"]; -var postgreSqlConnectionString = builder.Configuration.GetSection("PostgreSql")["ConnectionString"]; - -// Initialize Sentry -SentrySdk.Init(options => -{ - // A Sentry Data Source Name (DSN) is required. - options.Dsn = builder.Configuration["Sentry:Dsn"]; - - // When debug is enabled, the Sentry client will emit detailed debugging information to the console. - options.Debug = false; - - // Adds request URL and headers, IP and name for users, etc. - options.SendDefaultPii = true; - - // This option is recommended. It enables Sentry's "Release Health" feature. - options.AutoSessionTracking = true; - - // Enabling this option is recommended for client applications only. It ensures all threads use the same global scope. - options.IsGlobalModeEnabled = false; - - // Example sample rate for your transactions: captures 10% of transactions - options.TracesSampleRate = 0.1; - - options.Environment = builder.Environment.EnvironmentName; -}); - -// Add service discovery for Aspire -builder.Services.AddServiceDiscovery(); - -// Configure health checks -builder.Services.AddHealthChecks() - .AddCheck("self", () => HealthCheckResult.Healthy(), ["live"]) - .AddUrlGroup(new Uri($"{influxUrl}/health"), name: "influxdb", tags: ["database"]) - .AddUrlGroup(new Uri($"{web3ProxyUrl}/health"), name: "web3proxy", tags: ["api"]); - -builder.WebHost.UseUrls("http://localhost:5001"); - -builder.Host.UseSerilog((hostBuilder, loggerConfiguration) => -{ - var envName = builder.Environment.EnvironmentName.ToLower().Replace(".", "-"); - var indexFormat = $"managing-worker-{envName}-" + "{0:yyyy.MM.dd}"; - var yourTemplateName = "dotnetlogs"; - var es = new ElasticsearchSinkOptions(new Uri(hostBuilder.Configuration["ElasticConfiguration:Uri"])) - { - IndexFormat = indexFormat.ToLower(), - AutoRegisterTemplate = true, - OverwriteTemplate = true, - TemplateName = yourTemplateName, - AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7, - TypeName = null, - BatchAction = ElasticOpType.Create, - MinimumLogEventLevel = LogEventLevel.Information, - DetectElasticsearchVersion = true, - RegisterTemplateFailure = RegisterTemplateRecovery.IndexAnyway, - }; - - loggerConfiguration - .WriteTo.Console() - .WriteTo.Elasticsearch(es); -}); -builder.Services.AddOptions(); -builder.Services.Configure(builder.Configuration.GetSection(Constants.Databases.InfluxDb)); -builder.Services.Configure(builder.Configuration.GetSection(Constants.ThirdParty.Privy)); -builder.Services.AddControllers().AddJsonOptions(options => - options.JsonSerializerOptions.Converters.Add(new JsonStringEnumConverter())); -builder.Services.AddCors(o => o.AddPolicy("CorsPolicy", builder => -{ - builder - .SetIsOriginAllowed((host) => true) - .AllowAnyOrigin() - .WithOrigins("http://localhost:3000/") - .AllowAnyMethod() - .AllowAnyHeader() - .AllowCredentials(); -})); -builder.Services.AddSignalR().AddJsonProtocol(); - -// Add PostgreSQL DbContext for worker services -builder.Services.AddDbContext(options => -{ - options.UseNpgsql(postgreSqlConnectionString, npgsqlOptions => - { - npgsqlOptions.CommandTimeout(60); - npgsqlOptions.EnableRetryOnFailure(maxRetryCount: 5, maxRetryDelay: TimeSpan.FromSeconds(10), - errorCodesToAdd: null); - }); - if (builder.Environment.IsDevelopment()) - { - options.EnableDetailedErrors(); - options.EnableSensitiveDataLogging(); - options.EnableThreadSafetyChecks(); - } - - options.UseQueryTrackingBehavior(QueryTrackingBehavior.NoTracking); - options.EnableServiceProviderCaching(); - options.LogTo(msg => Console.WriteLine(msg), LogLevel.Warning); -}, ServiceLifetime.Scoped); - -builder.Services.RegisterWorkersDependencies(builder.Configuration); - -builder.Services.AddEndpointsApiExplorer(); -builder.Services.AddOpenApiDocument(document => -{ - document.AddSecurity("JWT", Enumerable.Empty(), new OpenApiSecurityScheme - { - Type = OpenApiSecuritySchemeType.ApiKey, - Name = "Authorization", - In = OpenApiSecurityApiKeyLocation.Header, - Description = "Type into the textbox: Bearer {your JWT token}." - }); - - document.OperationProcessors.Add( - new AspNetCoreOperationSecurityScopeProcessor("JWT")); -}); -builder.Services.AddSwaggerGen(options => -{ - options.SchemaFilter(); - options.AddSecurityDefinition("Bearer,", new Microsoft.OpenApi.Models.OpenApiSecurityScheme - { - Description = "Please insert your JWT Token into field : Bearer {your_token}", - Name = "Authorization", - Type = SecuritySchemeType.Http, - In = ParameterLocation.Header, - Scheme = "Bearer", - BearerFormat = "JWT" - }); - options.AddSecurityRequirement(new OpenApiSecurityRequirement - { - { - new Microsoft.OpenApi.Models.OpenApiSecurityScheme - { - Reference = new OpenApiReference - { - Type = ReferenceType.SecurityScheme, - Id = "Bearer" - } - }, - new string[] { } - } - }); -}); - -builder.WebHost.SetupDiscordBot(); - -// App -var app = builder.Build(); -app.UseSerilogRequestLogging(); - -app.UseOpenApi(); -app.UseSwaggerUI(c => -{ - c.SwaggerEndpoint("/swagger/v1/swagger.json", "Managing Workers v1"); - c.RoutePrefix = string.Empty; -}); - -app.UseCors("CorsPolicy"); - -// Add Sentry diagnostics middleware (now using shared version from Core) -app.UseSentryDiagnostics(); - -// Using shared GlobalErrorHandlingMiddleware from Core project -app.UseMiddleware(); - -app.UseHttpsRedirection(); - -app.UseRouting(); - -app.UseAuthorization(); - -app.UseEndpoints(endpoints => -{ - endpoints.MapControllers(); - endpoints.MapHub("/positionhub"); - - endpoints.MapHealthChecks("/health", new HealthCheckOptions - { - ResponseWriter = UIResponseWriter.WriteHealthCheckUIResponse - }); - - endpoints.MapHealthChecks("/alive", new HealthCheckOptions - { - Predicate = r => r.Tags.Contains("live"), - ResponseWriter = UIResponseWriter.WriteHealthCheckUIResponse - }); -}); - -app.Run(); \ No newline at end of file diff --git a/src/Managing.Api.Workers/appsettings.Development.json b/src/Managing.Api.Workers/appsettings.Development.json deleted file mode 100644 index 19023d0..0000000 --- a/src/Managing.Api.Workers/appsettings.Development.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "InfluxDb": { - "Url": "http://localhost:8086/", - "Token": "" - }, - "Serilog": { - "MinimumLevel": { - "Default": "Information", - "Override": { - "Microsoft": "Information", - "System": "Warning" - } - } - }, - "ElasticConfiguration": { - "Uri": "http://localhost:9200" - }, - "AllowedHosts": "*" -} \ No newline at end of file diff --git a/src/Managing.Api.Workers/appsettings.KaiServer.json b/src/Managing.Api.Workers/appsettings.KaiServer.json deleted file mode 100644 index 2a451c1..0000000 --- a/src/Managing.Api.Workers/appsettings.KaiServer.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "InfluxDb": { - "Url": "https://influx-db.apps.managing.live", - "Organization": "managing-org", - "Token": "eOuXcXhH7CS13Iw4CTiDDpRjIjQtEVPOloD82pLPOejI4n0BsEj1YzUw0g3Cs1mdDG5m-RaxCavCMsVTtS5wIQ==" - }, - "Privy": { - "AppId": "cm6f47n1l003jx7mjwaembhup", - "AppSecret": "63Chz2z5M8TgR5qc8dznSLRAGTHTyPU4cjdQobrBF1Cx5tszZpTuFgyrRd7hZ2k6HpwDz3GEwQZzsCqHb8Z311bF" - }, - "N8n": { - "WebhookUrl": "https://n8n.kai.managing.live/webhook/fa9308b6-983b-42ec-b085-71599d655951" - }, - "Serilog": { - "MinimumLevel": { - "Default": "Information", - "Override": { - "Microsoft": "Information", - "System": "Warning" - } - } - }, - "ElasticConfiguration": { - "Uri": "http://elasticsearch:9200" - }, - "AllowedHosts": "*" -} \ No newline at end of file diff --git a/src/Managing.Api.Workers/appsettings.Oda-docker.json b/src/Managing.Api.Workers/appsettings.Oda-docker.json deleted file mode 100644 index 5dee831..0000000 --- a/src/Managing.Api.Workers/appsettings.Oda-docker.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "InfluxDb": { - "Url": "http://influxdb:8086/", - "Organization": "managing-org", - "Token": "Fw2FPL2OwTzDHzSbR2Sd5xs0EKQYy00Q-hYKYAhr9cC1_q5YySONpxuf_Ck0PTjyUiF13xXmi__bu_pXH-H9zA==" - }, - "Privy": { - "AppId": "cm6f47n1l003jx7mjwaembhup", - "AppSecret": "63Chz2z5M8TgR5qc8dznSLRAGTHTyPU4cjdQobrBF1Cx5tszZpTuFgyrRd7hZ2k6HpwDz3GEwQZzsCqHb8Z311bF" - }, - "Serilog": { - "MinimumLevel": { - "Default": "Information", - "Override": { - "Microsoft": "Information", - "System": "Warning" - } - } - }, - "ElasticConfiguration": { - "Uri": "http://elasticsearch:9200" - }, - "AllowedHosts": "*" -} \ No newline at end of file diff --git a/src/Managing.Api.Workers/appsettings.Oda.json b/src/Managing.Api.Workers/appsettings.Oda.json deleted file mode 100644 index 5b499a3..0000000 --- a/src/Managing.Api.Workers/appsettings.Oda.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "PostgreSql": { - "ConnectionString": "Host=localhost;Port=5432;Database=managing;Username=postgres;Password=postgres", - "Orleans": "Host=localhost;Port=5432;Database=orleans;Username=postgres;Password=postgres" - }, - "InfluxDb": { - "Url": "http://localhost:8086/", - "Organization": "managing-org", - "Token": "Fw2FPL2OwTzDHzSbR2Sd5xs0EKQYy00Q-hYKYAhr9cC1_q5YySONpxuf_Ck0PTjyUiF13xXmi__bu_pXH-H9zA==" - }, - "Privy": { - "AppId": "cm6f47n1l003jx7mjwaembhup", - "AppSecret": "63Chz2z5M8TgR5qc8dznSLRAGTHTyPU4cjdQobrBF1Cx5tszZpTuFgyrRd7hZ2k6HpwDz3GEwQZzsCqHb8Z311bF" - }, - "Serilog": { - "MinimumLevel": { - "Default": "Information", - "Override": { - "Microsoft": "Information", - "System": "Warning" - } - } - }, - "ElasticConfiguration": { - "Uri": "http://localhost:9200" - }, - "AllowedHosts": "*", - "WorkerPricesFifteenMinutes": false, - "WorkerPricesOneHour": false, - "WorkerPricesFourHours": false, - "WorkerPricesOneDay": false, - "WorkerPricesFiveMinutes": false, - "WorkerFee": false, - "WorkerPositionManager": false, - "WorkerPositionFetcher": false, - "WorkerSpotlight": false, - "WorkerTraderWatcher": false, - "WorkerLeaderboard": false, - "WorkerFundingRatesWatcher": false, - "WorkerGeneticAlgorithm": false, - "WorkerBundleBacktest": true -} \ No newline at end of file diff --git a/src/Managing.Api.Workers/appsettings.Production.json b/src/Managing.Api.Workers/appsettings.Production.json deleted file mode 100644 index 8d961ee..0000000 --- a/src/Managing.Api.Workers/appsettings.Production.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "PostgreSql": { - "ConnectionString": "Host=managing-postgre.apps.managing.live;Port=5432;Database=managing;Username=postgres;Password=29032b13a5bc4d37", - "Orleans": "Host=managing-postgre.apps.managing.live;Port=5432;Database=orleans;Username=postgres;Password=29032b13a5bc4d37" - }, - "InfluxDb": { - "Url": "https://influx-db.apps.managing.live", - "Organization": "managing-org", - "Token": "_BtklT_aQ7GRqWG-HGILYEd8MJzxdbxxckPadzUsRofnwJBKQuXYLbCrVcLD7TrD4BlXgGAsyuqQItsOtanfBw==" - }, - "Privy": { - "AppId": "cm6kkz5ke00n5ffmpwdbr05mp", - "AppSecret": "3STq1UyPJ5WHixArBcVBKecWtyR4QpgZ1uju4HHvvJH2RwtacJnvoyzuaiNC8Xibi4rQb3eeH2YtncKrMxCYiV3a" - }, - "Web3Proxy": { - "BaseUrl": "http://srv-captain--managing-web3:4111" - }, - "Serilog": { - "MinimumLevel": { - "Default": "Information", - "Override": { - "Microsoft": "Information", - "System": "Warning" - } - } - }, - "ElasticConfiguration": { - "Uri": "http://elasticsearch:9200" - }, - "Sentry": { - "Dsn": "https://698e00d7cb404b049aff3881e5a47f6b@bugcenter.apps.managing.live/1" - }, - "AllowedHosts": "*" -} \ No newline at end of file diff --git a/src/Managing.Api.Workers/appsettings.Sandbox.json b/src/Managing.Api.Workers/appsettings.Sandbox.json deleted file mode 100644 index d1803f0..0000000 --- a/src/Managing.Api.Workers/appsettings.Sandbox.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "PostgreSql": { - "ConnectionString": "Host=managing-postgre.apps.managing.live;Port=5432;Database=managing;Username=postgres;Password=29032b13a5bc4d37" - }, - "InfluxDb": { - "Url": "http://srv-captain--influx-db:8086/", - "Organization": "managing-org", - "Token": "eOuXcXhH7CS13Iw4CTiDDpRjIjQtEVPOloD82pLPOejI4n0BsEj1YzUw0g3Cs1mdDG5m-RaxCavCMsVTtS5wIQ==" - }, - "Privy": { - "AppId": "cm6f47n1l003jx7mjwaembhup", - "AppSecret": "63Chz2z5M8TgR5qc8dznSLRAGTHTyPU4cjdQobrBF1Cx5tszZpTuFgyrRd7hZ2k6HpwDz3GEwQZzsCqHb8Z311bF" - }, - "Serilog": { - "MinimumLevel": { - "Default": "Information", - "Override": { - "Microsoft": "Information", - "System": "Warning" - } - } - }, - "ElasticConfiguration": { - "Uri": "http://elasticsearch:9200" - }, - "AllowedHosts": "*" -} \ No newline at end of file diff --git a/src/Managing.Api.Workers/appsettings.SandboxLocal.json b/src/Managing.Api.Workers/appsettings.SandboxLocal.json deleted file mode 100644 index ef99e1e..0000000 --- a/src/Managing.Api.Workers/appsettings.SandboxLocal.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "InfluxDb": { - "Url": "https://influx-db.apps.managing.live", - "Organization": "managing-org", - "Token": "eOuXcXhH7CS13Iw4CTiDDpRjIjQtEVPOloD82pLPOejI4n0BsEj1YzUw0g3Cs1mdDG5m-RaxCavCMsVTtS5wIQ==" - }, - "Privy": { - "AppId": "cm6f47n1l003jx7mjwaembhup", - "AppSecret": "63Chz2z5M8TgR5qc8dznSLRAGTHTyPU4cjdQobrBF1Cx5tszZpTuFgyrRd7hZ2k6HpwDz3GEwQZzsCqHb8Z311bF" - }, - "Serilog": { - "MinimumLevel": { - "Default": "Information", - "Override": { - "Microsoft": "Information", - "System": "Warning" - } - } - }, - "ElasticConfiguration": { - "Uri": "http://elasticsearch:9200" - }, - "AllowedHosts": "*" -} \ No newline at end of file diff --git a/src/Managing.Api.Workers/appsettings.json b/src/Managing.Api.Workers/appsettings.json deleted file mode 100644 index e0cd8d5..0000000 --- a/src/Managing.Api.Workers/appsettings.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "InfluxDb": { - "Url": "http://influxdb:8086/", - "Organization": "", - "Token": "" - }, - "PostgreSql": { - "ConnectionString": "Host=localhost;Port=5432;Database=managing;Username=postgres;Password=postgres" - }, - "Serilog": { - "MinimumLevel": { - "Default": "Information", - "Override": { - "Microsoft": "Information", - "System": "Warning" - } - } - }, - "N8n": { - "WebhookUrl": "https://n8n.kai.managing.live/webhook/fa9308b6-983b-42ec-b085-71599d655951" - }, - "ElasticConfiguration": { - "Uri": "http://elasticsearch:9200" - }, - "Web3Proxy": { - "BaseUrl": "http://localhost:4111" - }, - "Sentry": { - "Dsn": "https://8fdb299b69df4f9d9b709c8d4a556608@bugcenter.apps.managing.live/2", - "MinimumEventLevel": "Error", - "SendDefaultPii": true, - "MaxBreadcrumbs": 50, - "SampleRate": 1.0, - "TracesSampleRate": 0.2, - "Debug": false - }, - "Discord": { - "BotActivity": "with jobs", - "HandleUserAction": true, - "ApplicationId": "1132062339592622221", - "PublicKey": "e422f3326307788608eceba919497d3f2758cc64d20bb8a6504c695192404808", - "TokenId": "MTEzMjA2MjMzOTU5MjYyMjIyMQ.GySuNX.rU-9uIX6-yDthBjT_sbXioaJGyJva2ABNNEaj4", - "SignalChannelId": 966080506473099314, - "TradesChannelId": 998374177763491851, - "TroublesChannelId": 1015761955321040917, - "CopyTradingChannelId": 1132022887012909126, - "RequestsChannelId": 1018589494968078356, - "FundingRateChannelId": 1263566138709774336, - "LeaderboardChannelId": 1133169725237633095, - "ButtonExpirationMinutes": 10 - }, - "AllowedHosts": "*" -} \ No newline at end of file diff --git a/src/Managing.Api/README-ORLEANS-TROUBLESHOOTING.md b/src/Managing.Api/README-ORLEANS-TROUBLESHOOTING.md new file mode 100644 index 0000000..f249da3 --- /dev/null +++ b/src/Managing.Api/README-ORLEANS-TROUBLESHOOTING.md @@ -0,0 +1,118 @@ +# Orleans Clustering Troubleshooting Guide + +## Overview + +This document provides troubleshooting steps for Orleans clustering issues, particularly the "Connection attempt to endpoint failed" errors that can occur in Docker deployments. + +## Common Issues and Solutions + +### 1. Connection Timeout Errors + +**Error**: `Connection attempt to endpoint S10.0.0.9:11111:114298801 timed out after 00:00:05` + +**Cause**: Orleans silos are trying to connect to each other but the network configuration is preventing proper communication. + +**Solutions**: + +#### A. Environment Variables +You can disable Orleans clustering completely by setting: +```bash +DISABLE_ORLEANS_CLUSTERING=true +``` + +This will fall back to localhost clustering mode for testing. + +#### B. Docker Network Configuration +Ensure the Docker compose file includes proper network configuration: +```yaml +services: + managing.api: + ports: + - "11111:11111" # Orleans silo port + - "30000:30000" # Orleans gateway port + hostname: managing-api +``` + +#### C. Database Connection Issues +If the Orleans database is unavailable, the system will automatically fall back to: +- Localhost clustering +- Memory-based grain storage + +### 2. Configuration Options + +#### Production Settings +In `appsettings.Production.json`: +```json +{ + "RunOrleansGrains": true, + "Orleans": { + "EnableClustering": true, + "ConnectionTimeout": 60, + "MaxJoinAttempts": 3 + } +} +``` + +#### Environment Variables +- `RUN_ORLEANS_GRAINS`: Enable/disable Orleans grains (true/false) +- `DISABLE_ORLEANS_CLUSTERING`: Force localhost clustering (true/false) +- `ASPNETCORE_ENVIRONMENT`: Set environment (Production/Development/etc.) + +### 3. Network Configuration Improvements + +The following improvements have been made to handle Docker networking issues: + +1. **Endpoint Configuration**: + - `listenOnAnyHostAddress: true` allows binding to all network interfaces + - Increased timeout values for better reliability + +2. **Fallback Mechanisms**: + - Automatic fallback to localhost clustering if database unavailable + - Memory storage fallback for grain persistence + +3. **Improved Timeouts**: + - Response timeout: 60 seconds + - Probe timeout: 10 seconds + - Join attempt timeout: 120 seconds + +### 4. Monitoring and Debugging + +#### Orleans Dashboard +Available in development mode at: `http://localhost:9999` +- Username: admin +- Password: admin + +#### Health Checks +Monitor application health at: +- `/health` - Full health check +- `/alive` - Basic liveness check + +### 5. Emergency Procedures + +If Orleans clustering is causing deployment issues: + +1. **Immediate Fix**: Set environment variable `DISABLE_ORLEANS_CLUSTERING=true` +2. **Restart Services**: Restart the managing.api container +3. **Check Logs**: Monitor for connection timeout errors +4. **Database Check**: Verify PostgreSQL Orleans database connectivity + +### 6. Database Requirements + +Orleans requires these PostgreSQL databases: +- Main application database (from `PostgreSql:ConnectionString`) +- Orleans clustering database (from `PostgreSql:Orleans`) + +If either is unavailable, the system will gracefully degrade functionality. + +## Testing the Fix + +1. Deploy with the updated configuration +2. Monitor logs for Orleans connection errors +3. Verify grain functionality through the dashboard (development) or API endpoints +4. Test failover scenarios by temporarily disabling database connectivity + +## Related Files + +- `src/Managing.Bootstrap/ApiBootstrap.cs` - Orleans configuration +- `src/Managing.Docker/docker-compose.yml` - Docker networking +- `src/Managing.Api/appsettings.*.json` - Environment-specific settings diff --git a/src/Managing.Api/appsettings.Production.json b/src/Managing.Api/appsettings.Production.json index ac18080..5e33255 100644 --- a/src/Managing.Api/appsettings.Production.json +++ b/src/Managing.Api/appsettings.Production.json @@ -29,6 +29,11 @@ }, "RunOrleansGrains": true, "DeploymentMode": false, + "Orleans": { + "EnableClustering": true, + "ConnectionTimeout": 60, + "MaxJoinAttempts": 3 + }, "AllowedHosts": "*", "WorkerBotManager": true, "WorkerBalancesTracking": true, diff --git a/src/Managing.Bootstrap/ApiBootstrap.cs b/src/Managing.Bootstrap/ApiBootstrap.cs index 734dd07..43a629d 100644 --- a/src/Managing.Bootstrap/ApiBootstrap.cs +++ b/src/Managing.Bootstrap/ApiBootstrap.cs @@ -87,20 +87,33 @@ public static class ApiBootstrap runOrleansGrains = runOrleansGrainsFromEnv; } + // Allow disabling Orleans clustering entirely in case of issues + var disableOrleansClusteringEnv = Environment.GetEnvironmentVariable("DISABLE_ORLEANS_CLUSTERING"); + var disableOrleansClustering = !string.IsNullOrEmpty(disableOrleansClusteringEnv) && + bool.TryParse(disableOrleansClusteringEnv, out var disabled) && disabled; + var postgreSqlConnectionString = configuration.GetSection("PostgreSql")["Orleans"]; return hostBuilder.UseOrleans(siloBuilder => { - // Configure clustering with improved networking - siloBuilder - .UseAdoNetClustering(options => - { - options.ConnectionString = postgreSqlConnectionString; - options.Invariant = "Npgsql"; - }); + // Configure clustering with improved networking or use localhost clustering if disabled + if (!disableOrleansClustering && !string.IsNullOrEmpty(postgreSqlConnectionString)) + { + siloBuilder + .UseAdoNetClustering(options => + { + options.ConnectionString = postgreSqlConnectionString; + options.Invariant = "Npgsql"; + }); + } + else + { + // Fallback to localhost clustering for testing or when database is unavailable + siloBuilder.UseLocalhostClustering(); + } // Conditionally configure reminder service based on flag - if (runOrleansGrains) + if (runOrleansGrains && !disableOrleansClustering && !string.IsNullOrEmpty(postgreSqlConnectionString)) { siloBuilder.UseAdoNetReminderService(options => { @@ -111,7 +124,7 @@ public static class ApiBootstrap // Configure networking for better silo communication siloBuilder - .ConfigureEndpoints(siloPort: 11111, gatewayPort: 30000) + .ConfigureEndpoints(siloPort: 11111, gatewayPort: 30000, advertisedIP: null, listenOnAnyHostAddress: true) .Configure(options => { // Configure cluster options with unique identifiers @@ -120,8 +133,22 @@ public static class ApiBootstrap }) .Configure(options => { - // Configure messaging for better reliability - options.ResponseTimeout = TimeSpan.FromSeconds(30); + // Configure messaging for better reliability with increased timeouts + options.ResponseTimeout = TimeSpan.FromSeconds(60); + options.DropExpiredMessages = true; + }) + .Configure(options => + { + // Configure cluster membership for better resilience + options.EnableIndirectProbes = true; + options.ProbeTimeout = TimeSpan.FromSeconds(10); + options.IAmAliveTablePublishTimeout = TimeSpan.FromSeconds(30); + options.MaxJoinAttemptTime = TimeSpan.FromSeconds(120); + }) + .Configure(options => + { + // Configure gateway with improved timeouts + options.GatewayListRefreshPeriod = TimeSpan.FromSeconds(60); }); // Conditionally configure grain execution based on flag @@ -164,27 +191,42 @@ public static class ApiBootstrap }); } + // Configure grain storage - use ADO.NET for production or memory for fallback + if (!disableOrleansClustering && !string.IsNullOrEmpty(postgreSqlConnectionString)) + { + siloBuilder + .AddAdoNetGrainStorage("bot-store", options => + { + options.ConnectionString = postgreSqlConnectionString; + options.Invariant = "Npgsql"; + }) + .AddAdoNetGrainStorage("registry-store", options => + { + options.ConnectionString = postgreSqlConnectionString; + options.Invariant = "Npgsql"; + }) + .AddAdoNetGrainStorage("agent-store", options => + { + options.ConnectionString = postgreSqlConnectionString; + options.Invariant = "Npgsql"; + }) + .AddAdoNetGrainStorage("platform-summary-store", options => + { + options.ConnectionString = postgreSqlConnectionString; + options.Invariant = "Npgsql"; + }); + } + else + { + // Fallback to memory storage when database is unavailable + siloBuilder + .AddMemoryGrainStorage("bot-store") + .AddMemoryGrainStorage("registry-store") + .AddMemoryGrainStorage("agent-store") + .AddMemoryGrainStorage("platform-summary-store"); + } + siloBuilder - .AddAdoNetGrainStorage("bot-store", options => - { - options.ConnectionString = postgreSqlConnectionString; - options.Invariant = "Npgsql"; - }) - .AddAdoNetGrainStorage("registry-store", options => - { - options.ConnectionString = postgreSqlConnectionString; - options.Invariant = "Npgsql"; - }) - .AddAdoNetGrainStorage("agent-store", options => - { - options.ConnectionString = postgreSqlConnectionString; - options.Invariant = "Npgsql"; - }) - .AddAdoNetGrainStorage("platform-summary-store", options => - { - options.ConnectionString = postgreSqlConnectionString; - options.Invariant = "Npgsql"; - }) .ConfigureServices(services => { // Register existing services for Orleans DI diff --git a/src/Managing.Docker/docker-compose.yml b/src/Managing.Docker/docker-compose.yml index d7d5b65..ea8a522 100644 --- a/src/Managing.Docker/docker-compose.yml +++ b/src/Managing.Docker/docker-compose.yml @@ -8,6 +8,13 @@ services: dockerfile: Managing.Api/Dockerfile networks: - managing-network + ports: + - "11111:11111" # Orleans silo port + - "30000:30000" # Orleans gateway port + environment: + - ASPNETCORE_ENVIRONMENT=Production + - RUN_ORLEANS_GRAINS=true + hostname: managing-api influxdb: image: influxdb:latest @@ -20,10 +27,10 @@ services: - managing-network volumes: - mongodata: {} influxdata: {} postgresdata: {} networks: managing-network: - name: managing-network \ No newline at end of file + name: managing-network + driver: bridge \ No newline at end of file