Add synthApi (#27)

* Add synthApi * Put confidence for Synth proba * Update the code * Update readme * Fix bootstraping * fix github build * Update the endpoints for scenario * Add scenario and update backtest modal * Update bot modal * Update interfaces for synth * add synth to backtest * Add Kelly criterion and better signal * Update signal confidence * update doc * save leaderboard and prediction * Update nswag to generate ApiClient in the correct path * Unify the trading modal * Save miner and prediction * Update messaging and block new signal until position not close when flipping off * Rename strategies to indicators * Update doc * Update chart + add signal name * Fix signal direction * Update docker webui * remove crypto npm * Clean
2025-07-03 00:13:42 +07:00
parent 453806356d
commit a547c4a040
103 changed files with 9916 additions and 810 deletions
--- a/src/Managing.Infrastructure.Database/MongoDb/Collections/SignalDto.cs
+++ b/src/Managing.Infrastructure.Database/MongoDb/Collections/SignalDto.cs
@@ -18,5 +18,6 @@ namespace Managing.Infrastructure.Databases.MongoDb.Collections
        public IndicatorType Type { get; set; }
        public SignalType SignalType { get; set; }
        public UserDto User { get; set; }
+        public string IndicatorName { get; set; }
    }
 }
--- a/src/Managing.Infrastructure.Database/MongoDb/Collections/SynthMinersLeaderboardDto.cs
+++ b/src/Managing.Infrastructure.Database/MongoDb/Collections/SynthMinersLeaderboardDto.cs
@@ -0,0 +1,42 @@
+using Managing.Infrastructure.Databases.MongoDb.Attributes;
+using Managing.Infrastructure.Databases.MongoDb.Configurations;
+
+namespace Managing.Infrastructure.Databases.MongoDb.Collections;
+
+/// <summary>
+/// MongoDB DTO for storing Synth miners leaderboard data
+/// </summary>
+[BsonCollection("SynthMinersLeaderboard")]
+public class SynthMinersLeaderboardDto : Document
+{
+    /// <summary>
+    /// Asset symbol (e.g., "BTC", "ETH")
+    /// </summary>
+    public string Asset { get; set; }
+    
+    /// <summary>
+    /// Time increment used for this leaderboard data
+    /// </summary>
+    public int TimeIncrement { get; set; }
+    
+    /// <summary>
+    /// Signal date for which this leaderboard was retrieved (for backtests)
+    /// Null for live trading data
+    /// </summary>
+    public DateTime? SignalDate { get; set; }
+    
+    /// <summary>
+    /// Whether this is backtest data or live data
+    /// </summary>
+    public bool IsBacktest { get; set; }
+    
+    /// <summary>
+    /// Serialized JSON of miners list
+    /// </summary>
+    public string MinersData { get; set; }
+    
+    /// <summary>
+    /// Cache key for quick lookup
+    /// </summary>
+    public string CacheKey { get; set; }
+} 
--- a/src/Managing.Infrastructure.Database/MongoDb/Collections/SynthMinersPredictionsDto.cs
+++ b/src/Managing.Infrastructure.Database/MongoDb/Collections/SynthMinersPredictionsDto.cs
@@ -0,0 +1,57 @@
+using Managing.Infrastructure.Databases.MongoDb.Attributes;
+using Managing.Infrastructure.Databases.MongoDb.Configurations;
+
+namespace Managing.Infrastructure.Databases.MongoDb.Collections;
+
+/// <summary>
+/// MongoDB DTO for storing Synth miners predictions data
+/// </summary>
+[BsonCollection("SynthMinersPredictions")]
+public class SynthMinersPredictionsDto : Document
+{
+    /// <summary>
+    /// Asset symbol (e.g., "BTC", "ETH")
+    /// </summary>
+    public string Asset { get; set; }
+    
+    /// <summary>
+    /// Time increment used for these predictions
+    /// </summary>
+    public int TimeIncrement { get; set; }
+    
+    /// <summary>
+    /// Time length (horizon) for these predictions in seconds
+    /// </summary>
+    public int TimeLength { get; set; }
+    
+    /// <summary>
+    /// Signal date for which these predictions were retrieved (for backtests)
+    /// Null for live trading data
+    /// </summary>
+    public DateTime? SignalDate { get; set; }
+    
+    /// <summary>
+    /// Whether this is backtest data or live data
+    /// </summary>
+    public bool IsBacktest { get; set; }
+    
+    /// <summary>
+    /// Serialized JSON of miner UIDs list
+    /// </summary>
+    public string MinerUidsData { get; set; }
+    
+    /// <summary>
+    /// Serialized JSON of predictions data
+    /// </summary>
+    public string PredictionsData { get; set; }
+    
+    /// <summary>
+    /// When this prediction data was fetched from the API
+    /// </summary>
+    public DateTime FetchedAt { get; set; }
+    
+    /// <summary>
+    /// Cache key for quick lookup
+    /// </summary>
+    public string CacheKey { get; set; }
+} 
--- a/src/Managing.Infrastructure.Database/MongoDb/Collections/SynthPredictionDto.cs
+++ b/src/Managing.Infrastructure.Database/MongoDb/Collections/SynthPredictionDto.cs
@@ -0,0 +1,52 @@
+using Managing.Infrastructure.Databases.MongoDb.Attributes;
+using Managing.Infrastructure.Databases.MongoDb.Configurations;
+
+namespace Managing.Infrastructure.Databases.MongoDb.Collections;
+
+/// <summary>
+/// MongoDB DTO for storing individual Synth miner prediction data
+/// </summary>
+[BsonCollection("SynthPredictions")]
+public class SynthPredictionDto : Document
+{
+    /// <summary>
+    /// Asset symbol (e.g., "BTC", "ETH")
+    /// </summary>
+    public string Asset { get; set; }
+    
+    /// <summary>
+    /// Miner UID that provided this prediction
+    /// </summary>
+    public int MinerUid { get; set; }
+    
+    /// <summary>
+    /// Time increment used for this prediction
+    /// </summary>
+    public int TimeIncrement { get; set; }
+    
+    /// <summary>
+    /// Time length (horizon) for this prediction in seconds
+    /// </summary>
+    public int TimeLength { get; set; }
+    
+    /// <summary>
+    /// Signal date for which this prediction was retrieved (for backtests)
+    /// Null for live trading data
+    /// </summary>
+    public DateTime? SignalDate { get; set; }
+    
+    /// <summary>
+    /// Whether this is backtest data or live data
+    /// </summary>
+    public bool IsBacktest { get; set; }
+    
+    /// <summary>
+    /// Serialized JSON of the prediction data
+    /// </summary>
+    public string PredictionData { get; set; }
+    
+    /// <summary>
+    /// Cache key for quick lookup
+    /// </summary>
+    public string CacheKey { get; set; }
+} 
--- a/src/Managing.Infrastructure.Database/MongoDb/MongoMappers.cs
+++ b/src/Managing.Infrastructure.Database/MongoDb/MongoMappers.cs
@@ -1,4 +1,5 @@
-using Managing.Domain.Accounts;
+using System.Text.Json;
+using Managing.Domain.Accounts;
 using Managing.Domain.Backtests;
 using Managing.Domain.Bots;
 using Managing.Domain.Candles;
@@ -6,6 +7,7 @@ using Managing.Domain.MoneyManagements;
 using Managing.Domain.Scenarios;
 using Managing.Domain.Statistics;
 using Managing.Domain.Strategies;
+using Managing.Domain.Synth.Models;
 using Managing.Domain.Trades;
 using Managing.Domain.Users;
 using Managing.Domain.Workers;
@@ -371,7 +373,8 @@ public static class MongoMappers
            Status = signal.Status,
            Timeframe = signal.Timeframe,
            Type = signal.IndicatorType,
-            User = signal.User != null ? Map(signal.User) : null
+            User = signal.User != null ? Map(signal.User) : null,
+            IndicatorName = signal.IndicatorName
        };
    }

@@ -387,6 +390,7 @@ public static class MongoMappers
            TradingExchanges.Binance, //TODO FIXME When the signal status is modified from controller
            bSignal.Type,
            bSignal.SignalType,
+            bSignal.IndicatorName,
            bSignal.User != null ? Map(bSignal.User) : null)
        {
            Status = bSignal.Status
@@ -744,7 +748,6 @@ public static class MongoMappers
        {
            User = Map(bot.User),
            Identifier = bot.Identifier,
-            BotType = bot.BotType,
            Data = bot.Data,
            LastStatus = bot.LastStatus
        };
@@ -758,7 +761,6 @@ public static class MongoMappers
        {
            User = Map(b.User),
            Identifier = b.Identifier,
-            BotType = b.BotType,
            Data = b.Data,
            LastStatus = b.LastStatus
        };
@@ -792,4 +794,143 @@ public static class MongoMappers
            Direction = fundingRate.Direction
        };
    }
+
+    #region Synth
+
+    /// <summary>
+    /// Maps domain SynthMinersLeaderboard to MongoDB DTO
+    /// </summary>
+    internal static SynthMinersLeaderboardDto Map(SynthMinersLeaderboard leaderboard)
+    {
+        if (leaderboard == null) return null;
+
+        return new SynthMinersLeaderboardDto
+        {
+            Asset = leaderboard.Asset,
+            TimeIncrement = leaderboard.TimeIncrement,
+            SignalDate = leaderboard.SignalDate,
+            IsBacktest = leaderboard.IsBacktest,
+            MinersData = JsonSerializer.Serialize(leaderboard.Miners),
+            CacheKey = leaderboard.GetCacheKey()
+        };
+    }
+
+    /// <summary>
+    /// Maps MongoDB DTO to domain SynthMinersLeaderboard
+    /// </summary>
+    internal static SynthMinersLeaderboard Map(SynthMinersLeaderboardDto dto)
+    {
+        if (dto == null) return null;
+
+        var miners = string.IsNullOrEmpty(dto.MinersData)
+            ? new List<MinerInfo>()
+            : JsonSerializer.Deserialize<List<MinerInfo>>(dto.MinersData) ?? new List<MinerInfo>();
+
+        return new SynthMinersLeaderboard
+        {
+            Id = dto.Id.ToString(),
+            Asset = dto.Asset,
+            TimeIncrement = dto.TimeIncrement,
+            SignalDate = dto.SignalDate,
+            IsBacktest = dto.IsBacktest,
+            Miners = miners,
+            CreatedAt = dto.CreatedAt
+        };
+    }
+
+    /// <summary>
+    /// Maps domain SynthMinersPredictions to MongoDB DTO
+    /// </summary>
+    internal static SynthMinersPredictionsDto Map(SynthMinersPredictions predictions)
+    {
+        if (predictions == null) return null;
+
+        return new SynthMinersPredictionsDto
+        {
+            Asset = predictions.Asset,
+            TimeIncrement = predictions.TimeIncrement,
+            TimeLength = predictions.TimeLength,
+            SignalDate = predictions.SignalDate,
+            IsBacktest = predictions.IsBacktest,
+            MinerUidsData = JsonSerializer.Serialize(predictions.MinerUids),
+            PredictionsData = JsonSerializer.Serialize(predictions.Predictions),
+            CacheKey = predictions.GetCacheKey()
+        };
+    }
+
+    /// <summary>
+    /// Maps MongoDB DTO to domain SynthMinersPredictions
+    /// </summary>
+    internal static SynthMinersPredictions Map(SynthMinersPredictionsDto dto)
+    {
+        if (dto == null) return null;
+
+        var minerUids = string.IsNullOrEmpty(dto.MinerUidsData)
+            ? new List<int>()
+            : JsonSerializer.Deserialize<List<int>>(dto.MinerUidsData) ?? new List<int>();
+
+        var predictions = string.IsNullOrEmpty(dto.PredictionsData)
+            ? new List<MinerPrediction>()
+            : JsonSerializer.Deserialize<List<MinerPrediction>>(dto.PredictionsData) ?? new List<MinerPrediction>();
+
+        return new SynthMinersPredictions
+        {
+            Id = dto.Id.ToString(),
+            Asset = dto.Asset,
+            TimeIncrement = dto.TimeIncrement,
+            TimeLength = dto.TimeLength,
+            SignalDate = dto.SignalDate,
+            IsBacktest = dto.IsBacktest,
+            MinerUids = minerUids,
+            Predictions = predictions,
+            CreatedAt = dto.CreatedAt
+        };
+    }
+
+    /// <summary>
+    /// Maps domain SynthPrediction to MongoDB DTO
+    /// </summary>
+    internal static SynthPredictionDto Map(SynthPrediction prediction)
+    {
+        if (prediction == null) return null;
+
+        return new SynthPredictionDto
+        {
+            Asset = prediction.Asset,
+            MinerUid = prediction.MinerUid,
+            TimeIncrement = prediction.TimeIncrement,
+            TimeLength = prediction.TimeLength,
+            SignalDate = prediction.SignalDate,
+            IsBacktest = prediction.IsBacktest,
+            PredictionData = JsonSerializer.Serialize(prediction.Prediction),
+            CacheKey = prediction.GetCacheKey()
+        };
+    }
+
+    /// <summary>
+    /// Maps MongoDB DTO to domain SynthPrediction
+    /// </summary>
+    internal static SynthPrediction Map(SynthPredictionDto dto)
+    {
+        if (dto == null) return null;
+
+        var prediction = string.IsNullOrEmpty(dto.PredictionData)
+            ? null
+            : JsonSerializer.Deserialize<MinerPrediction>(dto.PredictionData);
+
+        return new SynthPrediction
+        {
+            Id = dto.Id.ToString(),
+            Asset = dto.Asset,
+            MinerUid = dto.MinerUid,
+            TimeIncrement = dto.TimeIncrement,
+            TimeLength = dto.TimeLength,
+            SignalDate = dto.SignalDate,
+            IsBacktest = dto.IsBacktest,
+            Prediction = prediction,
+            CreatedAt = dto.CreatedAt
+        };
+    }
+
+    #endregion
 }
--- a/src/Managing.Infrastructure.Database/SynthRepository.cs
+++ b/src/Managing.Infrastructure.Database/SynthRepository.cs
@@ -0,0 +1,223 @@
+using Managing.Application.Abstractions.Repositories;
+using Managing.Domain.Synth.Models;
+using Managing.Infrastructure.Databases.MongoDb;
+using Managing.Infrastructure.Databases.MongoDb.Abstractions;
+using Managing.Infrastructure.Databases.MongoDb.Collections;
+using Microsoft.Extensions.Logging;
+
+namespace Managing.Infrastructure.Databases;
+
+/// <summary>
+/// Repository implementation for Synth-related data operations using MongoDB
+/// Provides persistence for leaderboard and individual predictions data
+/// </summary>
+public class SynthRepository : ISynthRepository
+{
+    private readonly IMongoRepository<SynthMinersLeaderboardDto> _leaderboardRepository;
+    private readonly IMongoRepository<SynthPredictionDto> _individualPredictionsRepository;
+    private readonly ILogger<SynthRepository> _logger;
+
+    public SynthRepository(
+        IMongoRepository<SynthMinersLeaderboardDto> leaderboardRepository,
+        IMongoRepository<SynthPredictionDto> individualPredictionsRepository,
+        ILogger<SynthRepository> logger)
+    {
+        _leaderboardRepository = leaderboardRepository;
+        _individualPredictionsRepository = individualPredictionsRepository;
+        _logger = logger;
+    }
+
+    /// <summary>
+    /// Gets cached leaderboard data by cache key
+    /// </summary>
+    public async Task<SynthMinersLeaderboard?> GetLeaderboardAsync(string cacheKey)
+    {
+        try
+        {
+            var dto = await _leaderboardRepository.FindOneAsync(x => x.CacheKey == cacheKey);
+
+            if (dto == null)
+            {
+                _logger.LogDebug($"🔍 **Synth Cache** - No leaderboard cache found for key: {cacheKey}");
+                return null;
+            }
+
+            var result = MongoMappers.Map(dto);
+            _logger.LogDebug($"📦 **Synth Cache** - Retrieved leaderboard from MongoDB for key: {cacheKey}");
+            
+            return result;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, $"Error retrieving leaderboard cache for key: {cacheKey}");
+            return null;
+        }
+    }
+
+    /// <summary>
+    /// Saves leaderboard data to MongoDB
+    /// </summary>
+    public async Task SaveLeaderboardAsync(SynthMinersLeaderboard leaderboard)
+    {
+        try
+        {
+            leaderboard.CreatedAt = DateTime.UtcNow;
+            var dto = MongoMappers.Map(leaderboard);
+            
+            // Check if we already have this cache key and update instead of inserting
+            var existing = await _leaderboardRepository.FindOneAsync(x => x.CacheKey == dto.CacheKey);
+            if (existing != null)
+            {
+                dto.Id = existing.Id;
+                _leaderboardRepository.Update(dto);
+                _logger.LogDebug($"💾 **Synth Cache** - Updated leaderboard in MongoDB for key: {dto.CacheKey}");
+            }
+            else
+            {
+                await _leaderboardRepository.InsertOneAsync(dto);
+                _logger.LogDebug($"💾 **Synth Cache** - Saved new leaderboard to MongoDB for key: {dto.CacheKey}");
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, $"Error saving leaderboard cache for key: {leaderboard.GetCacheKey()}");
+        }
+    }
+
+    /// <summary>
+    /// Gets individual cached prediction data by asset, parameters, and miner UIDs
+    /// </summary>
+    public async Task<List<SynthPrediction>> GetIndividualPredictionsAsync(
+        string asset,
+        int timeIncrement,
+        int timeLength,
+        List<int> minerUids,
+        bool isBacktest,
+        DateTime? signalDate)
+    {
+        try
+        {
+            var results = new List<SynthPrediction>();
+
+            foreach (var minerUid in minerUids)
+            {
+                // Build cache key for individual prediction
+                var cacheKey = $"{asset}_{timeIncrement}_{timeLength}_{minerUid}";
+                if (isBacktest && signalDate.HasValue)
+                {
+                    cacheKey += $"_backtest_{signalDate.Value:yyyy-MM-dd-HH}";
+                }
+
+                var dto = await _individualPredictionsRepository.FindOneAsync(x => x.CacheKey == cacheKey);
+
+                if (dto != null)
+                {
+                    var prediction = MongoMappers.Map(dto);
+                    if (prediction != null)
+                    {
+                        results.Add(prediction);
+                    }
+                }
+            }
+
+            if (results.Any())
+            {
+                _logger.LogDebug($"📦 **Synth Individual Cache** - Retrieved {results.Count}/{minerUids.Count} individual predictions for {asset}");
+            }
+            else
+            {
+                _logger.LogDebug($"🔍 **Synth Individual Cache** - No individual predictions found for {asset}");
+            }
+
+            return results;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, $"Error retrieving individual predictions cache for asset: {asset}");
+            return new List<SynthPrediction>();
+        }
+    }
+
+    /// <summary>
+    /// Saves individual prediction data to MongoDB
+    /// </summary>
+    public async Task SaveIndividualPredictionAsync(SynthPrediction prediction)
+    {
+        try
+        {
+            prediction.CreatedAt = DateTime.UtcNow;
+            var dto = MongoMappers.Map(prediction);
+
+            // Check if we already have this cache key and update instead of inserting
+            var existing = await _individualPredictionsRepository.FindOneAsync(x => x.CacheKey == dto.CacheKey);
+            if (existing != null)
+            {
+                dto.Id = existing.Id;
+                _individualPredictionsRepository.Update(dto);
+                _logger.LogDebug($"💾 **Synth Individual Cache** - Updated individual prediction for miner {prediction.MinerUid}");
+            }
+            else
+            {
+                await _individualPredictionsRepository.InsertOneAsync(dto);
+                _logger.LogDebug($"💾 **Synth Individual Cache** - Saved new individual prediction for miner {prediction.MinerUid}");
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, $"Error saving individual prediction cache for miner {prediction.MinerUid}: {ex.Message}");
+        }
+    }
+
+    /// <summary>
+    /// Saves multiple individual predictions to MongoDB in batch
+    /// </summary>
+    public async Task SaveIndividualPredictionsAsync(List<SynthPrediction> predictions)
+    {
+        if (!predictions.Any())
+        {
+            return;
+        }
+
+        try
+        {
+            var saveTasks = new List<Task>();
+
+            foreach (var prediction in predictions)
+            {
+                // Save each prediction individually to handle potential conflicts
+                saveTasks.Add(SaveIndividualPredictionAsync(prediction));
+            }
+
+            await Task.WhenAll(saveTasks);
+
+            _logger.LogInformation($"💾 **Synth Individual Cache** - Successfully saved {predictions.Count} individual predictions to MongoDB");
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, $"Error saving batch of {predictions.Count} individual predictions");
+        }
+    }
+
+    /// <summary>
+    /// Cleans up old cached data beyond the retention period
+    /// </summary>
+    public async Task CleanupOldDataAsync(int retentionDays = 30)
+    {
+        try
+        {
+            var cutoffDate = DateTime.UtcNow.AddDays(-retentionDays);
+            
+            // Clean up old leaderboard data
+            await _leaderboardRepository.DeleteManyAsync(x => x.CreatedAt < cutoffDate);
+            
+            // Clean up old individual predictions data
+            await _individualPredictionsRepository.DeleteManyAsync(x => x.CreatedAt < cutoffDate);
+            
+            _logger.LogInformation($"🧹 **Synth Cache** - Cleaned up old Synth cache data older than {retentionDays} days");
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, $"Error during cleanup of old Synth cache data");
+        }
+    }
+}