From 9d0c7cf834b0f91f22f2f7bc858d6efd1ce327fa Mon Sep 17 00:00:00 2001 From: cryptooda Date: Wed, 13 Aug 2025 22:22:22 +0700 Subject: [PATCH] Fix bots restart/stop --- .../Bots/Grains/LiveTradingBotGrain.cs | 169 +++++++++++++----- 1 file changed, 121 insertions(+), 48 deletions(-) diff --git a/src/Managing.Application/Bots/Grains/LiveTradingBotGrain.cs b/src/Managing.Application/Bots/Grains/LiveTradingBotGrain.cs index 6f41fc1..cd34ecb 100644 --- a/src/Managing.Application/Bots/Grains/LiveTradingBotGrain.cs +++ b/src/Managing.Application/Bots/Grains/LiveTradingBotGrain.cs @@ -103,7 +103,7 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable private async Task ResumeBotInternalAsync() { - // The core of this method remains idempotent thanks to the _tradingBot null check + // Idempotency check if (_tradingBot != null) { return; @@ -111,21 +111,28 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable try { - // Load state from persisted grain state + // Create and initialize trading bot instance _tradingBot = CreateTradingBotInstance(_state.State.Config); - LoadStateIntoBase(); await _tradingBot.Start(); + // Set startup time when bot actually starts running + _state.State.StartupTime = DateTime.UtcNow; + await _state.WriteStateAsync(); + // Start the in-memory timer and persistent reminder RegisterAndStartTimer(); await RegisterReminder(); + + // Update both database and registry status await SaveBotAsync(BotStatus.Up); + await UpdateBotRegistryStatus(BotStatus.Up); + _logger.LogInformation("LiveTradingBotGrain {GrainId} resumed successfully", this.GetPrimaryKey()); } catch (Exception ex) { _logger.LogError(ex, "Failed to resume bot {GrainId}", this.GetPrimaryKey()); - // If resume fails, update the status to Down via the registry and stop + _tradingBot = null; // Clean up on failure await UpdateBotRegistryStatus(BotStatus.Down); throw; } @@ -137,7 +144,7 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable var botId = this.GetPrimaryKey(); var status = await botRegistry.GetBotStatus(botId); - // This is the new idempotency check, using the registry as the source of truth + // Check if already running if (status == BotStatus.Up && _tradingBot != null) { await RegisterReminder(); @@ -147,17 +154,14 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable try { - // Resume the bot using the internal logic + // Resume the bot - this handles registry status update internally await ResumeBotInternalAsync(); - - // Update registry status (if it was previously 'Down') - await UpdateBotRegistryStatus(BotStatus.Up); _logger.LogInformation("LiveTradingBotGrain {GrainId} started successfully", this.GetPrimaryKey()); } catch (Exception ex) { _logger.LogError(ex, "Failed to start LiveTradingBotGrain {GrainId}", this.GetPrimaryKey()); - // Ensure registry status is correct even on failure + // Ensure registry status is correct on failure await UpdateBotRegistryStatus(BotStatus.Down); throw; } @@ -253,7 +257,17 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable var logger = scope.ServiceProvider.GetRequiredService>(); var tradingBot = new TradingBotBase(logger, _scopeFactory, config); - // Restore state from grain state + // Load state into the trading bot instance + LoadStateIntoTradingBot(tradingBot); + + return tradingBot; + } + + /// + /// Loads grain state into a trading bot instance + /// + private void LoadStateIntoTradingBot(TradingBotBase tradingBot) + { tradingBot.Signals = _state.State.Signals; tradingBot.Positions = _state.State.Positions; tradingBot.WalletBalances = _state.State.WalletBalances; @@ -262,8 +276,6 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable tradingBot.Identifier = _state.State.Identifier; tradingBot.LastPositionClosingTime = _state.State.LastPositionClosingTime; tradingBot.LastCandle = _state.State.LastCandle; - - return tradingBot; } @@ -340,13 +352,28 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable { if (_tradingBot == null) { - throw new InvalidOperationException("Bot is not running"); + // For non-running bots, return data from grain state only + return Task.FromResult(new LiveTradingBotModel + { + Identifier = _state.State.Identifier, + Name = _state.State.Config?.Name ?? "Unknown", + Config = _state.State.Config, + Positions = _state.State.Positions ?? new Dictionary(), + Signals = _state.State.Signals, + WalletBalances = _state.State.WalletBalances ?? new Dictionary(), + ProfitAndLoss = 0, // Calculate from persisted positions if needed + WinRate = 0, // Calculate from persisted positions if needed + ExecutionCount = _state.State.ExecutionCount, + StartupTime = _state.State.StartupTime, + CreateDate = _state.State.CreateDate + }); } + // For running bots, return live data return Task.FromResult(new LiveTradingBotModel { Identifier = _state.State.Identifier, - Name = _state.State.Name, + Name = _state.State.Config?.Name ?? "Unknown", Config = _state.State.Config, Positions = _tradingBot.Positions, Signals = _tradingBot.Signals, @@ -365,22 +392,6 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable } } - private void LoadStateIntoBase() - { - if (_tradingBot == null) - _tradingBot = CreateTradingBotInstance(_state.State.Config); - - if (_tradingBot == null) throw new InvalidOperationException("TradingBotBase instance could not be created"); - - _tradingBot.Signals = _state.State.Signals; - _tradingBot.Positions = _state.State.Positions; - _tradingBot.WalletBalances = _state.State.WalletBalances; - _tradingBot.PreloadedCandlesCount = _state.State.PreloadedCandlesCount; - _tradingBot.ExecutionCount = _state.State.ExecutionCount; - _tradingBot.Identifier = _state.State.Identifier; - _tradingBot.LastPositionClosingTime = _state.State.LastPositionClosingTime; - _tradingBot.LastCandle = _state.State.LastCandle; - } private void SyncStateFromBase() { @@ -399,15 +410,23 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable public async Task UpdateConfiguration(TradingBotConfig newConfig) { if (_tradingBot == null) - LoadStateIntoBase(); + { + // For non-running bots, just update the configuration + _state.State.Config = newConfig; + await _state.WriteStateAsync(); + + var botRegistry = GrainFactory.GetGrain(0); + var status = await botRegistry.GetBotStatus(this.GetPrimaryKey()); + await SaveBotAsync(status); + return true; + } - var result = await _tradingBot!.UpdateConfiguration(newConfig); + var result = await _tradingBot.UpdateConfiguration(newConfig); if (result) { var botRegistry = GrainFactory.GetGrain(0); - var botId = this.GetPrimaryKey(); - var status = await botRegistry.GetBotStatus(botId); + var status = await botRegistry.GetBotStatus(this.GetPrimaryKey()); _state.State.Config = newConfig; await _state.WriteStateAsync(); await SaveBotAsync(status); @@ -418,6 +437,10 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable public Task GetAccount() { + if (_tradingBot == null) + { + throw new InvalidOperationException("Bot is not running - cannot get account information"); + } return Task.FromResult(_tradingBot.Account); } @@ -453,8 +476,29 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable public async Task RestartAsync() { - await StopAsync(); - await StartAsync(); + _logger.LogInformation("Restarting LiveTradingBotGrain {GrainId}", this.GetPrimaryKey()); + + try + { + await StopAsync(); + + // Add a small delay to ensure stop operations complete + await Task.Delay(100); + + await StartAsync(); + + // Verify the restart was successful + var botRegistry = GrainFactory.GetGrain(0); + var finalStatus = await botRegistry.GetBotStatus(this.GetPrimaryKey()); + + _logger.LogInformation("LiveTradingBotGrain {GrainId} restart completed with final status: {Status}", + this.GetPrimaryKey(), finalStatus); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to restart LiveTradingBotGrain {GrainId}", this.GetPrimaryKey()); + throw; + } } public async Task DeleteAsync() @@ -489,21 +533,50 @@ public class LiveTradingBotGrain : Grain, ILiveTradingBotGrain, IRemindable } /// - /// Updates the bot status in the central BotRegistry + /// Updates the bot status in the central BotRegistry with retry logic /// private async Task UpdateBotRegistryStatus(BotStatus status) { - try + const int maxRetries = 3; + var botId = this.GetPrimaryKey(); + + for (int attempt = 1; attempt <= maxRetries; attempt++) { - var botRegistry = GrainFactory.GetGrain(0); - var botId = this.GetPrimaryKey(); - await botRegistry.UpdateBotStatus(botId, status); - _logger.LogDebug("Bot {BotId} status updated to {Status} in BotRegistry", botId, status); - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to update bot {BotId} status to {Status} in BotRegistry", this.GetPrimaryKey(), - status); + try + { + var botRegistry = GrainFactory.GetGrain(0); + await botRegistry.UpdateBotStatus(botId, status); + + // Verify the update was successful + var actualStatus = await botRegistry.GetBotStatus(botId); + if (actualStatus == status) + { + _logger.LogDebug("Bot {BotId} status successfully updated to {Status} in BotRegistry (attempt {Attempt})", + botId, status, attempt); + return; + } + else + { + _logger.LogWarning("Bot {BotId} status update verification failed. Expected: {Expected}, Actual: {Actual} (attempt {Attempt})", + botId, status, actualStatus, attempt); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to update bot {BotId} status to {Status} in BotRegistry (attempt {Attempt})", + botId, status, attempt); + + if (attempt == maxRetries) + { + throw; + } + } + + // Wait before retry + if (attempt < maxRetries) + { + await Task.Delay(TimeSpan.FromMilliseconds(100 * attempt)); + } } }