pagination for backtest and optimization

src/Managing.Api.Workers/Program.cs

@@ -6,6 +6,7 @@ using Managing.Bootstrap;
 using Managing.Common;
 using Managing.Core.Middleawares;
 using Managing.Infrastructure.Databases.InfluxDb.Models;
+using Managing.Infrastructure.Databases.MongoDb;
 using Managing.Infrastructure.Databases.MongoDb.Configurations;
 using Managing.Infrastructure.Evm.Models.Privy;
 using Microsoft.AspNetCore.Diagnostics.HealthChecks;
@@ -156,6 +157,19 @@ builder.WebHost.SetupDiscordBot();
 var app = builder.Build();
 app.UseSerilogRequestLogging();
 
+// Create MongoDB indexes on startup
+try
+{
+    var indexService = app.Services.GetRequiredService<IndexService>();
+    await indexService.CreateIndexesAsync();
+}
+catch (Exception ex)
+{
+    // Log the error but don't fail the application startup
+    var logger = app.Services.GetRequiredService<ILogger<Program>>();
+    logger.LogError(ex, "Failed to create MongoDB indexes on startup. The application will continue without indexes.");
+}
+
 if (app.Environment.IsDevelopment())
 {
     app.UseDeveloperExceptionPage();

src/Managing.Api/Controllers/BacktestController.cs

@@ -134,6 +134,54 @@ public class BacktestController : BaseController
         return Ok(backtests);
     }
 
+    /// <summary>
+    /// Retrieves paginated backtests for a specific genetic request ID.
+    /// This endpoint is used to view the results of a genetic algorithm optimization with pagination support.
+    /// </summary>
+    /// <param name="requestId">The request ID to filter backtests by.</param>
+    /// <param name="page">Page number (defaults to 1)</param>
+    /// <param name="pageSize">Number of items per page (defaults to 50, max 100)</param>
+    /// <returns>A paginated list of backtests associated with the specified request ID.</returns>
+    [HttpGet]
+    [Route("ByRequestId/{requestId}/Paginated")]
+    public async Task<ActionResult<PaginatedBacktestsResponse>> GetBacktestsByRequestIdPaginated(
+        string requestId, 
+        int page = 1, 
+        int pageSize = 50)
+    {
+        if (string.IsNullOrEmpty(requestId))
+        {
+            return BadRequest("Request ID is required");
+        }
+
+        if (page < 1)
+        {
+            return BadRequest("Page must be greater than 0");
+        }
+
+        if (pageSize < 1 || pageSize > 100)
+        {
+            return BadRequest("Page size must be between 1 and 100");
+        }
+
+        var (backtests, totalCount) = _backtester.GetBacktestsByRequestIdPaginated(requestId, page, pageSize);
+        
+        var totalPages = (int)Math.Ceiling(totalCount / (double)pageSize);
+        
+        var response = new PaginatedBacktestsResponse
+        {
+            Backtests = backtests,
+            TotalCount = totalCount,
+            CurrentPage = page,
+            PageSize = pageSize,
+            TotalPages = totalPages,
+            HasNextPage = page < totalPages,
+            HasPreviousPage = page > 1
+        };
+
+        return Ok(response);
+    }
+
     /// <summary>
     /// Runs a backtest with the specified configuration.
     /// The returned backtest includes a complete TradingBotConfig that preserves all

src/Managing.Api/Program.cs

@@ -10,6 +10,7 @@ using Managing.Bootstrap;
 using Managing.Common;
 using Managing.Core.Middleawares;
 using Managing.Infrastructure.Databases.InfluxDb.Models;
+using Managing.Infrastructure.Databases.MongoDb;
 using Managing.Infrastructure.Databases.MongoDb.Configurations;
 using Managing.Infrastructure.Evm.Models.Privy;
 using Microsoft.AspNetCore.Authentication.JwtBearer;
@@ -209,6 +210,19 @@ if (builder.Configuration.GetValue<bool>("EnableBotManager", false))
 var app = builder.Build();
 app.UseSerilogRequestLogging();
 
+// Create MongoDB indexes on startup
+try
+{
+    var indexService = app.Services.GetRequiredService<IndexService>();
+    await indexService.CreateIndexesAsync();
+}
+catch (Exception ex)
+{
+    // Log the error but don't fail the application startup
+    var logger = app.Services.GetRequiredService<ILogger<Program>>();
+    logger.LogError(ex, "Failed to create MongoDB indexes on startup. The application will continue without indexes.");
+}
+
 if (app.Environment.IsDevelopment())
 {
     app.UseDeveloperExceptionPage();

src/Managing.Api/appsettings.json

@@ -28,7 +28,7 @@
     "BaseUrl": "https://api.kaigen.managing.live",
     "DebitEndpoint": "/api/credits/debit",
     "RefundEndpoint": "/api/credits/refund",
-    "PrivateKey": "${KAIGEN_PRIVATE_KEY}"
+    "PrivateKey": "0x0fb7fbebde2b9a14b039fa974ad330dd693f91e783cd4ea13ed38be8706835a7"
   },
   "N8n": {
     "WebhookUrl": "https://n8n.kai.managing.live/webhook/fa9308b6-983b-42ec-b085-71599d655951"

src/Managing.Application.Abstractions/Repositories/IBacktestRepository.cs

@@ -8,6 +8,7 @@ public interface IBacktestRepository
     void InsertBacktestForUser(User user, Backtest result);
     IEnumerable<Backtest> GetBacktestsByUser(User user);
     IEnumerable<Backtest> GetBacktestsByRequestId(string requestId);
+    (IEnumerable<Backtest> Backtests, int TotalCount) GetBacktestsByRequestIdPaginated(string requestId, int page, int pageSize);
     Backtest GetBacktestByIdForUser(User user, string id);
     void DeleteBacktestByIdForUser(User user, string id);
     void DeleteBacktestsByIdsForUser(User user, IEnumerable<string> ids);

src/Managing.Application.Abstractions/Services/IBacktester.cs

@@ -54,6 +54,7 @@ namespace Managing.Application.Abstractions.Services
         bool DeleteBacktests();
         IEnumerable<Backtest> GetBacktestsByUser(User user);
         IEnumerable<Backtest> GetBacktestsByRequestId(string requestId);
+        (IEnumerable<Backtest> Backtests, int TotalCount) GetBacktestsByRequestIdPaginated(string requestId, int page, int pageSize);
         Backtest GetBacktestByIdForUser(User user, string id);
         bool DeleteBacktestByUser(User user, string id);
         bool DeleteBacktestsByIdsForUser(User user, IEnumerable<string> ids);

src/Managing.Application/Backtesting/Backtester.cs

@@ -448,6 +448,12 @@ namespace Managing.Application.Backtesting
             return backtests;
         }
 
+        public (IEnumerable<Backtest> Backtests, int TotalCount) GetBacktestsByRequestIdPaginated(string requestId, int page, int pageSize)
+        {
+            var (backtests, totalCount) = _backtestRepository.GetBacktestsByRequestIdPaginated(requestId, page, pageSize);
+            return (backtests, totalCount);
+        }
+
         public Backtest GetBacktestByIdForUser(User user, string id)
         {
             var backtest = _backtestRepository.GetBacktestByIdForUser(user, id);

src/Managing.Bootstrap/ApiBootstrap.cs

@@ -149,6 +149,13 @@ public static class ApiBootstrap
         services.AddTransient<ICacheService, CacheService>();
         services.AddSingleton<ITaskCache, TaskCache>();
 
+        // Index Service
+        services.AddSingleton<IndexService>();
+
+        // Services
+        services.AddTransient<ICacheService, CacheService>();
+        services.AddSingleton<ITaskCache, TaskCache>();
+
         return services;
     }
 

src/Managing.Bootstrap/WorkersBootstrap.cs

@@ -181,6 +181,9 @@ public static class WorkersBootstrap
         services.AddTransient<ICacheService, CacheService>();
         services.AddTransient<ITaskCache, TaskCache>();
 
+        // Index Service
+        services.AddSingleton<IndexService>();
+
         // Processors
         services.AddTransient<IExchangeProcessor, EvmProcessor>();
 

src/Managing.Infrastructure.Database/BacktestRepository.cs

@@ -1,9 +1,11 @@
-using Managing.Application.Abstractions.Repositories;
+using System.Diagnostics;
+using Managing.Application.Abstractions.Repositories;
 using Managing.Domain.Backtests;
 using Managing.Domain.Users;
 using Managing.Infrastructure.Databases.MongoDb;
 using Managing.Infrastructure.Databases.MongoDb.Abstractions;
 using Managing.Infrastructure.Databases.MongoDb.Collections;
+using MongoDB.Driver;
 
 namespace Managing.Infrastructure.Databases;
 
@@ -41,6 +43,48 @@ public class BacktestRepository : IBacktestRepository
         return backtests.Select(b => MongoMappers.Map(b));
     }
 
+    public (IEnumerable<Backtest> Backtests, int TotalCount) GetBacktestsByRequestIdPaginated(string requestId, int page, int pageSize)
+    {
+        var stopwatch = Stopwatch.StartNew();
+        var collection = _backtestRepository.GetCollection(); // You may need to expose this in your repo
+
+        var filter = Builders<BacktestDto>.Filter.Eq(b => b.RequestId, requestId);
+
+        var afterQueryMs = stopwatch.ElapsedMilliseconds;
+        var totalCount = collection.CountDocuments(filter);
+        var afterCountMs = stopwatch.ElapsedMilliseconds;
+
+        var projection = Builders<BacktestDto>.Projection
+            .Include(b => b.Identifier)
+            .Include(b => b.FinalPnl)
+            .Include(b => b.WinRate)
+            .Include(b => b.GrowthPercentage)
+            .Include(b => b.HodlPercentage)
+            .Include(b => b.User)
+            .Include(b => b.Statistics)
+            .Include(b => b.StartDate)
+            .Include(b => b.EndDate)
+            .Include(b => b.Score)
+            .Include(b => b.RequestId)
+            .Include(b => b.Metadata)
+            .Include(b => b.Config);
+
+        var afterProjectionMs = stopwatch.ElapsedMilliseconds;
+        var backtests = collection
+            .Find(filter)
+            .Project<BacktestDto>(projection)
+            .Skip((page - 1) * pageSize)
+            .Limit(pageSize)
+            .ToList();
+        var afterToListMs = stopwatch.ElapsedMilliseconds;
+
+        Console.WriteLine($"[BacktestRepo] Query: {afterQueryMs}ms, Count: {afterCountMs - afterQueryMs}ms, Projection: {afterProjectionMs - afterCountMs}ms, ToList: {afterToListMs - afterProjectionMs}ms, Total: {afterToListMs}ms");
+
+        var mappedBacktests = backtests.Select(b => MongoMappers.Map(b));
+        
+        return (mappedBacktests, (int)totalCount);
+    }
+
     public Backtest GetBacktestByIdForUser(User user, string id)
     {
         var backtest = _backtestRepository.FindById(id);

src/Managing.Infrastructure.Database/MongoDb/Abstractions/IMongoRepository.cs

@@ -1,6 +1,6 @@
-using Managing.Infrastructure.Databases.MongoDb.Configurations;
+using System.Linq.Expressions;
+using Managing.Infrastructure.Databases.MongoDb.Configurations;
 using MongoDB.Driver;
-using System.Linq.Expressions;
 
 namespace Managing.Infrastructure.Databases.MongoDb.Abstractions
 {
@@ -53,5 +53,6 @@ namespace Managing.Infrastructure.Databases.MongoDb.Abstractions
         void Update(TDocument entity);
         void CreateIndex(string column);
         void DropCollection();
+        IMongoCollection<TDocument> GetCollection();
     }
 }

src/Managing.Infrastructure.Database/MongoDb/MongoRepository.cs

@@ -1,9 +1,9 @@
-using Managing.Infrastructure.Databases.MongoDb.Abstractions;
+using System.Linq.Expressions;
+using Managing.Infrastructure.Databases.MongoDb.Abstractions;
 using Managing.Infrastructure.Databases.MongoDb.Attributes;
 using Managing.Infrastructure.Databases.MongoDb.Configurations;
 using MongoDB.Bson;
 using MongoDB.Driver;
-using System.Linq.Expressions;
 
 namespace Managing.Infrastructure.Databases.MongoDb
 {
@@ -174,5 +174,10 @@ namespace Managing.Infrastructure.Databases.MongoDb
             var model = new CreateIndexModel<TDocument>(keys, indexOptions);
             _collection.Indexes.CreateOne(model);
         }
+
+        public IMongoCollection<TDocument> GetCollection()
+        {
+            return _collection;
+        }
     }
 }

src/Managing.WebApp/src/generated/ManagingApi.ts

@@ -580,6 +580,52 @@ export class BacktestClient extends AuthorizedApiBase {
         return Promise.resolve<Backtest[]>(null as any);
     }
 
+    backtest_GetBacktestsByRequestIdPaginated(requestId: string, page: number | undefined, pageSize: number | undefined): Promise<PaginatedBacktestsResponse> {
+        let url_ = this.baseUrl + "/Backtest/ByRequestId/{requestId}/Paginated?";
+        if (requestId === undefined || requestId === null)
+            throw new Error("The parameter 'requestId' must be defined.");
+        url_ = url_.replace("{requestId}", encodeURIComponent("" + requestId));
+        if (page === null)
+            throw new Error("The parameter 'page' cannot be null.");
+        else if (page !== undefined)
+            url_ += "page=" + encodeURIComponent("" + page) + "&";
+        if (pageSize === null)
+            throw new Error("The parameter 'pageSize' cannot be null.");
+        else if (pageSize !== undefined)
+            url_ += "pageSize=" + encodeURIComponent("" + pageSize) + "&";
+        url_ = url_.replace(/[?&]$/, "");
+
+        let options_: RequestInit = {
+            method: "GET",
+            headers: {
+                "Accept": "application/json"
+            }
+        };
+
+        return this.transformOptions(options_).then(transformedOptions_ => {
+            return this.http.fetch(url_, transformedOptions_);
+        }).then((_response: Response) => {
+            return this.processBacktest_GetBacktestsByRequestIdPaginated(_response);
+        });
+    }
+
+    protected processBacktest_GetBacktestsByRequestIdPaginated(response: Response): Promise<PaginatedBacktestsResponse> {
+        const status = response.status;
+        let _headers: any = {}; if (response.headers && response.headers.forEach) { response.headers.forEach((v: any, k: any) => _headers[k] = v); };
+        if (status === 200) {
+            return response.text().then((_responseText) => {
+            let result200: any = null;
+            result200 = _responseText === "" ? null : JSON.parse(_responseText, this.jsonParseReviver) as PaginatedBacktestsResponse;
+            return result200;
+            });
+        } else if (status !== 200 && status !== 204) {
+            return response.text().then((_responseText) => {
+            return throwException("An unexpected server error occurred.", status, _responseText, _headers);
+            });
+        }
+        return Promise.resolve<PaginatedBacktestsResponse>(null as any);
+    }
+
     backtest_Run(request: RunBacktestRequest): Promise<Backtest> {
         let url_ = this.baseUrl + "/Backtest/Run";
         url_ = url_.replace(/[?&]$/, "");
@@ -3696,6 +3742,16 @@ export interface DeleteBacktestsRequest {
     backtestIds: string[];
 }
 
+export interface PaginatedBacktestsResponse {
+    backtests?: Backtest[] | null;
+    totalCount?: number;
+    currentPage?: number;
+    pageSize?: number;
+    totalPages?: number;
+    hasNextPage?: boolean;
+    hasPreviousPage?: boolean;
+}
+
 export interface RunBacktestRequest {
     config?: TradingBotConfigRequest | null;
     startDate?: Date;

src/Managing.WebApp/src/generated/ManagingApiTypes.ts

@@ -599,6 +599,16 @@ export interface DeleteBacktestsRequest {
     backtestIds: string[];
 }
 
+export interface PaginatedBacktestsResponse {
+    backtests?: Backtest[] | null;
+    totalCount?: number;
+    currentPage?: number;
+    pageSize?: number;
+    totalPages?: number;
+    hasNextPage?: boolean;
+    hasPreviousPage?: boolean;
+}
+
 export interface RunBacktestRequest {
     config?: TradingBotConfigRequest | null;
     startDate?: Date;

src/Managing.WebApp/src/pages/backtestPage/backtestGeneticBundle.tsx

@@ -11,6 +11,7 @@ import {
   type GeneticRequest,
   GeneticSelectionMethod,
   IndicatorType,
+  type PaginatedBacktestsResponse,
   type RunGeneticRequest,
   Ticker,
   Timeframe,
@@ -76,6 +77,12 @@ const BacktestGeneticBundle: React.FC = () => {
     const [isLoadingBacktests, setIsLoadingBacktests] = useState(false)
     const [isFormCollapsed, setIsFormCollapsed] = useState(false)
     
+    // Pagination state
+    const [currentPage, setCurrentPage] = useState(1)
+    const [pageSize, setPageSize] = useState(50)
+    const [totalBacktests, setTotalBacktests] = useState(0)
+    const [totalPages, setTotalPages] = useState(0)
+
     // Form setup
     const {register, handleSubmit, watch, setValue, formState: {errors}} = useForm<GeneticBundleFormData>({
         defaultValues: {
@@ -219,9 +226,20 @@ const BacktestGeneticBundle: React.FC = () => {
         setIsViewModalOpen(true)
         setIsLoadingBacktests(true)
         
+        // Reset pagination state
+        setCurrentPage(1)
+        setTotalBacktests(0)
+        setTotalPages(0)
+
         try {
-            const backtestsData = await backtestClient.backtest_GetBacktestsByRequestId(request.requestId)
+            const response: PaginatedBacktestsResponse = await backtestClient.backtest_GetBacktestsByRequestIdPaginated(
-            setBacktests(backtestsData)
+                request.requestId, 
+                1, 
+                pageSize
+            )
+            setBacktests(response.backtests || [])
+            setTotalBacktests(response.totalCount || 0)
+            setTotalPages(response.totalPages || 0)
         } catch (error) {
             console.error('Error fetching backtests:', error)
             new Toast('Failed to load backtest details', false)
@@ -235,6 +253,31 @@ const BacktestGeneticBundle: React.FC = () => {
         setIsViewModalOpen(false)
         setSelectedRequest(null)
         setBacktests([])
+        setCurrentPage(1)
+        setTotalBacktests(0)
+        setTotalPages(0)
+    }
+
+    // Handle page change
+    const handlePageChange = async (newPage: number) => {
+        if (!selectedRequest || newPage < 1 || newPage > totalPages) return
+        
+        setIsLoadingBacktests(true)
+        setCurrentPage(newPage)
+
+        try {
+            const response: PaginatedBacktestsResponse = await backtestClient.backtest_GetBacktestsByRequestIdPaginated(
+                selectedRequest.requestId, 
+                newPage, 
+                pageSize
+            )
+            setBacktests(response.backtests || [])
+        } catch (error) {
+            console.error('Error fetching backtests:', error)
+            new Toast('Failed to load backtest details', false)
+        } finally {
+            setIsLoadingBacktests(false)
+        }
     }
 
     // Table columns for genetic requests
@@ -698,7 +741,32 @@ const BacktestGeneticBundle: React.FC = () => {
                         </div>
 
                         <div className="mb-6">
-                            <h4 className="font-semibold mb-2">Backtest Results ({backtests.length})</h4>
+                            <div className="flex justify-between items-center mb-4">
+                                <h4 className="font-semibold">Backtest Results ({totalBacktests} total)</h4>
+                                {totalPages > 1 && (
+                                    <div className="flex items-center gap-2">
+                                        <span className="text-sm text-gray-600">
+                                            Page {currentPage} of {totalPages}
+                                        </span>
+                                        <div className="join">
+                                            <button
+                                                className="join-item btn btn-sm"
+                                                onClick={() => handlePageChange(currentPage - 1)}
+                                                disabled={currentPage <= 1 || isLoadingBacktests}
+                                            >
+                                                «
+                                            </button>
+                                            <button
+                                                className="join-item btn btn-sm"
+                                                onClick={() => handlePageChange(currentPage + 1)}
+                                                disabled={currentPage >= totalPages || isLoadingBacktests}
+                                            >
+                                                »
+                                            </button>
+                                        </div>
+                                    </div>
+                                )}
+                            </div>
                             {isLoadingBacktests ? (
                                 <div className="flex justify-center">
                                     <span className="loading loading-spinner loading-md"></span>